diff options
Diffstat (limited to 'src')
80 files changed, 5016 insertions, 0 deletions
diff --git a/src/connectors/.classpath b/src/connectors/.classpath new file mode 100644 index 0000000..e421f8f --- /dev/null +++ b/src/connectors/.classpath @@ -0,0 +1,21 @@ +<?xml version="1.0" encoding="UTF-8"?> +<classpath> + <classpathentry kind="src" path="src"/> + <classpathentry kind="src" path="JUTests"/> + <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/> + <classpathentry combineaccessrules="false" kind="src" path="/SSSync_Core"/> + <classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/> + <classpathentry kind="lib" path="lib/commons-csv-1.0-SNAPSHOT.jar"/> + <classpathentry kind="lib" path="lib/ojdbc6.jar"> + <attributes> + <attribute name="javadoc_location" value="jar:platform:/resource/SSSync_Connectors/lib/ojdbc6-javadoc.jar!/"/> + </attributes> + </classpathentry> + <classpathentry kind="lib" path="lib/mysql-connector-java-5.1.31-bin.jar"/> + <classpathentry kind="lib" path="lib/unboundid-ldapsdk-se.jar"> + <attributes> + <attribute name="javadoc_location" value="jar:platform:/resource/SSSync_Connectors/lib/unboundid-ldapsdk-se-javadoc.jar!/"/> + </attributes> + </classpathentry> + <classpathentry kind="output" path="bin"/> +</classpath> diff --git a/src/connectors/.project b/src/connectors/.project new file mode 100644 index 0000000..b4f50df --- /dev/null +++ b/src/connectors/.project @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="UTF-8"?> +<projectDescription> + <name>SSSync_Connectors</name> + <comment></comment> + <projects> + </projects> + <buildSpec> + <buildCommand> + <name>org.eclipse.jdt.core.javabuilder</name> + <arguments> + </arguments> + </buildCommand> + </buildSpec> + <natures> + <nature>org.eclipse.jdt.core.javanature</nature> + </natures> +</projectDescription> diff --git a/src/connectors/.settings/org.eclipse.jdt.core.prefs b/src/connectors/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..8000cd6 --- /dev/null +++ b/src/connectors/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,11 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=1.6 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.source=1.6 diff --git a/src/connectors/JUTests/data/io/csv/CSVDataReaderTest.java b/src/connectors/JUTests/data/io/csv/CSVDataReaderTest.java new file mode 100644 index 0000000..6a0e053 --- /dev/null +++ b/src/connectors/JUTests/data/io/csv/CSVDataReaderTest.java @@ -0,0 +1,50 @@ +package data.io.csv; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Iterator; + +import org.junit.Test; + +import data.MVDataEntry; + +public class CSVDataReaderTest { + + + @Test + public void testNext() throws IOException { + CSVDataReader reader = new CSVDataReader( + "testNext", + new StringReader(CSVDataReader.CSV_DEMO), + false + ); + + MVDataEntry expected[] = new MVDataEntry[3]; + expected[0]=new MVDataEntry("line1"); + expected[0].splitAndPut("from", "csv1;csv1bis", ";"); + expected[0].splitAndPut("attr2","csv1",";"); + + expected[1]=new MVDataEntry("line2"); + expected[1].splitAndPut("hello", "all;the;world", ";"); + + expected[2]=new MVDataEntry("line3"); + expected[2].splitAndPut("hello", "all;the;others", ";"); + + // Test twice to check if asking a new iterator "rewinds" correctly + for (int i=0;i<2;i++) { + System.out.println("Loop " + (i+1)); + Iterator<MVDataEntry> readerIt = reader.iterator(); + + for ( MVDataEntry e: expected) { + assertTrue(readerIt.hasNext()); + MVDataEntry r = readerIt.next(); + System.out.println(e + " / " + r); + assertEquals(e, r); + } + assertFalse(readerIt.hasNext()); + } + } + +} diff --git a/src/connectors/JUTests/data/io/ldap/LDAPDataReaderTest.java b/src/connectors/JUTests/data/io/ldap/LDAPDataReaderTest.java new file mode 100644 index 0000000..dcfc602 --- /dev/null +++ b/src/connectors/JUTests/data/io/ldap/LDAPDataReaderTest.java @@ -0,0 +1,94 @@ +package data.io.ldap; + +import static org.junit.Assert.assertTrue; + +import org.junit.Before; +import org.junit.Test; + +import data.MVDataEntry; + +public class LDAPDataReaderTest { + + LDAPConnectionWrapper builder; + + @Before + public void setup() { + builder = new LDAPConnectionWrapper("localhost", 389, "uid=ldapadmin,ou=specialUsers,dc=univ-jfc,dc=fr", "secret"); + } + + /* + @Test + public void testLookAhead1() { + _testLookAhead(1); + } + */ + + @Test + public void testLookAhead16() { + _testLookAhead(16); + } + + @Test + public void testLookAhead32() { + _testLookAhead(32); + } + + @Test + public void testLookAhead64() { + _testLookAhead(64); + } + + @Test + public void testLookAhead128() { + _testLookAhead(128); + } + + @Test + public void testLookAhead192() { + _testLookAhead(192); + } + + @Test + public void testLookAhead256() { + _testLookAhead(256); + } + + @Test + public void testLookAhead512() { + _testLookAhead(512); + } + + @Test + public void testLookAhead1024() { + _testLookAhead(1024); + } + + private void _testLookAhead(int lookAheadAmount) { + System.out.println("_testLookAhead("+lookAheadAmount+")"); + LDAPFlatDataReader reader = builder.newFlatReader("ldap_test", "ou=people,dc=univ-jfc,dc=fr", "uid", lookAheadAmount); + + int resultCount = 0; + String previousKey=null; + for ( MVDataEntry entry : reader ) { + //System.out.println(entry); + if ( previousKey != null ) assertTrue(entry.getKey().compareTo(previousKey) > 0); + resultCount++; + previousKey=entry.getKey(); + } + System.out.println(resultCount); + assertTrue(resultCount>0); + + // Second time with a second iterator (must give the same results) + int newResultCount = 0; + previousKey=null; + for ( MVDataEntry entry : reader ) { + //System.out.println(entry); + if ( previousKey != null ) assertTrue(entry.getKey().compareTo(previousKey) > 0); + newResultCount++; + previousKey=entry.getKey(); + } + System.out.println(newResultCount); + assertTrue(newResultCount == resultCount); + + } +} diff --git a/src/connectors/JUTests/data/io/ldap/LDAPDataWriterTest.java b/src/connectors/JUTests/data/io/ldap/LDAPDataWriterTest.java new file mode 100644 index 0000000..01a8af0 --- /dev/null +++ b/src/connectors/JUTests/data/io/ldap/LDAPDataWriterTest.java @@ -0,0 +1,16 @@ +package data.io.ldap; + +import static org.junit.Assert.*; + +import org.junit.Test; + +public class LDAPDataWriterTest { + + @Test + public void test() { + fail("Not yet implemented"); + } + + // TODO : test update() extensively : null, empty string, add/update/delete subcases... + +} diff --git a/src/connectors/JUTests/data/io/sql/SQLRelDataReaderTest.java b/src/connectors/JUTests/data/io/sql/SQLRelDataReaderTest.java new file mode 100644 index 0000000..a97a98d --- /dev/null +++ b/src/connectors/JUTests/data/io/sql/SQLRelDataReaderTest.java @@ -0,0 +1,115 @@ +package data.io.sql; + +import static org.junit.Assert.*; + +import java.io.File; +import java.io.IOException; +import java.net.URL; + +import org.junit.Before; +import org.junit.Test; + +import data.MVDataEntry; +import data.io.MVDataReader; +import data.io.sql.SQLConnectionWrapper.DBMSType; + + +/* + +CREATE TABLE sssync.people ( + uid CHAR(16) NULL , + uidNumber INT NOT NULL , + gidNumber INT NULL , + cn VARCHAR(45) NULL , + sn VARCHAR(45) NULL , + homeDirectory VARCHAR(45) NULL , + PRIMARY KEY (uid) ); +INSERT INTO sssync.people (uid, uidNumber, gidNumber, cn, sn, homeDirectory) VALUES ('lpouzenc', 1000, 999, 'Ludovic', 'Pouzenc', '/home/lpouzenc'); +INSERT INTO sssync.people (uid, uidNumber, gidNumber, cn, sn, homeDirectory) VALUES ('dpouzenc', 1001, 999, 'Daniel', 'Pouzenc', '/home/dpouzenc'); + + +for i in $(seq 10000 20000); do echo "INSERT INTO sssync.people (uid, uidNumber, gidNumber, cn, sn, homeDirectory) VALUES ('test$i', $i, 999, '$i', 'test', '/home/test$i');"; done | mysql -uroot -p + + + +DROP TABLE IF EXISTS structures; +CREATE TABLE structures ( + supannCodeEntite varchar(15) NOT NULL, + ou varchar(45) NOT NULL, + supannTypeEntite varchar(45) NOT NULL, + supannCodeEntiteParent varchar(45) NOT NULL, + PRIMARY KEY (supannCodeEntite) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +INSERT INTO structures VALUES ('2','CUFR','Etablissement','2'),('9','Personnels','Groupe','2'); + + + +TODO : make automated tests with embded Derby base + + */ +public class SQLRelDataReaderTest { + + private static final String TEST_REQUEST = "SELECT p.*, \"person;posixAccount;top\" as objectClass" + + " FROM sssync.people p" + + " ORDER BY 1 ASC;"; + + private SQLConnectionWrapper builder; + private MVDataReader reader1; + private MVDataReader reader2; + + @Before + public void setup() throws IOException { + // Find the folder containing this test class + URL main = SQLRelDataReaderTest.class.getResource("SQLRelDataReaderTest.class"); + if (!"file".equalsIgnoreCase(main.getProtocol())) + throw new IllegalStateException("This class is not stored in a file"); + File currentFolder = new File(main.getPath()).getParentFile(); + + // Build a connection and two readers on it + builder = new SQLConnectionWrapper(DBMSType.mysql, "localhost", 3306, null, "root", "secret", "sssync"); + reader1 = builder.newReader("testMysql1", TEST_REQUEST); + reader2 = builder.newReader("testMysql2", new File(currentFolder, "req_test.sql")); + } + + @Test + public void testNext() { + // First full read on reader1 + int resultCount_r1i1 = 0; + String previousKey_r1i1=null; + for ( MVDataEntry entry : reader1 ) { + //System.out.println(entry); + if ( previousKey_r1i1 != null ) assertTrue(entry.getKey().compareTo(previousKey_r1i1) > 0); + resultCount_r1i1++; + previousKey_r1i1=entry.getKey(); + } + System.out.println(resultCount_r1i1); + assertTrue(resultCount_r1i1 > 0); + + // First half read on reader2 + int resultCount_r2i1 = 0; + String previousKey_r2i1=null; + for ( MVDataEntry entry : reader2 ) { + //System.out.println(entry); + if ( previousKey_r2i1 != null ) assertTrue(entry.getKey().compareTo(previousKey_r2i1) > 0); + resultCount_r2i1++; + previousKey_r2i1=entry.getKey(); + if ( resultCount_r2i1 > resultCount_r1i1 / 2 ) break; + } + System.out.println(resultCount_r2i1); + assertTrue(resultCount_r2i1 > resultCount_r1i1 / 2 ); + + // Second time with a second iterator on reader1 (must give the same results than r1i1) + int resultCount_r1i2 = 0; + String previousKey_r1i2=null; + for ( MVDataEntry entry : reader1 ) { + //System.out.println(entry); + if ( previousKey_r1i2 != null ) assertTrue(entry.getKey().compareTo(previousKey_r1i2) > 0); + resultCount_r1i2++; + previousKey_r1i2=entry.getKey(); + } + System.out.println(resultCount_r1i2); + assertTrue(resultCount_r1i2 == resultCount_r1i1); + } + +} diff --git a/src/connectors/JUTests/data/io/sql/req_test.sql b/src/connectors/JUTests/data/io/sql/req_test.sql new file mode 100644 index 0000000..ab66d5f --- /dev/null +++ b/src/connectors/JUTests/data/io/sql/req_test.sql @@ -0,0 +1,5 @@ +SELECT + p.*, + "person;posixAccount;top" as objectClass +FROM sssync.people p +ORDER BY 1 ASC; diff --git a/src/connectors/build.xml b/src/connectors/build.xml new file mode 100644 index 0000000..fdae9de --- /dev/null +++ b/src/connectors/build.xml @@ -0,0 +1,89 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- WARNING: Eclipse auto-generated file. + Any modifications will be overwritten. + To include a user specific buildfile here, simply create one in the same + directory with the processing instruction <?eclipse.ant.import?> + as the first entry and export the buildfile again. --> +<project basedir="." default="build" name="SSSync_Connectors"> + <property environment="env"/> + <property name="SSSync_Main.location" value="../main"/> + <property name="ECLIPSE_HOME" value="../../../../../../usr/lib/eclipse"/> + <property name="SSSync_Core.location" value="../core"/> + <property name="debuglevel" value="source,lines,vars"/> + <property name="target" value="1.6"/> + <property name="source" value="1.6"/> + <path id="JUnit 4.libraryclasspath"> + <pathelement location="../../../../../../usr/share/eclipse/dropins/jdt/plugins/org.junit_4.8.2.dist/junit.jar"/> + <pathelement location="../../../../../../usr/share/eclipse/dropins/jdt/plugins/org.hamcrest.core_1.1.0.jar"/> + </path> + <path id="SSSync_Core.classpath"> + <pathelement location="${SSSync_Core.location}/bin"/> + <pathelement location="${SSSync_Core.location}/lib/guava-16.0.1.jar"/> + <path refid="JUnit 4.libraryclasspath"/> + </path> + <path id="SSSync_Connectors.classpath"> + <pathelement location="bin"/> + <path refid="SSSync_Core.classpath"/> + <path refid="JUnit 4.libraryclasspath"/> + <pathelement location="lib/commons-csv-1.0-SNAPSHOT.jar"/> + <pathelement location="lib/ojdbc6.jar"/> + <pathelement location="lib/mysql-connector-java-5.1.31-bin.jar"/> + <pathelement location="lib/unboundid-ldapsdk-se.jar"/> + </path> + <target name="init"> + <mkdir dir="bin"/> + <copy includeemptydirs="false" todir="bin"> + <fileset dir="src"> + <exclude name="**/*.java"/> + </fileset> + </copy> + <copy includeemptydirs="false" todir="bin"> + <fileset dir="JUTests"> + <exclude name="**/*.java"/> + </fileset> + </copy> + </target> + <target name="clean"> + <delete dir="bin"/> + </target> + <target depends="clean" name="cleanall"> + <ant antfile="build.xml" dir="${SSSync_Core.location}" inheritAll="false" target="clean"/> + </target> + <target depends="build-subprojects,build-project" name="build"/> + <target name="build-subprojects"> + <ant antfile="build.xml" dir="${SSSync_Core.location}" inheritAll="false" target="build-project"> + <propertyset> + <propertyref name="build.compiler"/> + </propertyset> + </ant> + </target> + <target depends="init" name="build-project"> + <echo message="${ant.project.name}: ${ant.file}"/> + <javac debug="true" debuglevel="${debuglevel}" destdir="bin" includeantruntime="false" source="${source}" target="${target}"> + <src path="src"/> + <src path="JUTests"/> + <classpath refid="SSSync_Connectors.classpath"/> + </javac> + </target> + <target description="Build all projects which reference this project. Useful to propagate changes." name="build-refprojects"> + <ant antfile="build.xml" dir="${SSSync_Main.location}" inheritAll="false" target="clean"/> + <ant antfile="build.xml" dir="${SSSync_Main.location}" inheritAll="false" target="build"> + <propertyset> + <propertyref name="build.compiler"/> + </propertyset> + </ant> + </target> + <target description="copy Eclipse compiler jars to ant lib directory" name="init-eclipse-compiler"> + <copy todir="${ant.library.dir}"> + <fileset dir="${ECLIPSE_HOME}/plugins" includes="org.eclipse.jdt.core_*.jar"/> + </copy> + <unzip dest="${ant.library.dir}"> + <patternset includes="jdtCompilerAdapter.jar"/> + <fileset dir="${ECLIPSE_HOME}/plugins" includes="org.eclipse.jdt.core_*.jar"/> + </unzip> + </target> + <target description="compile project with Eclipse compiler" name="build-eclipse-compiler"> + <property name="build.compiler" value="org.eclipse.jdt.core.JDTCompilerAdapter"/> + <antcall target="build"/> + </target> +</project> diff --git a/src/connectors/lib/commons-csv-1.0-SNAPSHOT.jar b/src/connectors/lib/commons-csv-1.0-SNAPSHOT.jar Binary files differnew file mode 100644 index 0000000..f6a74f1 --- /dev/null +++ b/src/connectors/lib/commons-csv-1.0-SNAPSHOT.jar diff --git a/src/connectors/lib/derby.jar b/src/connectors/lib/derby.jar Binary files differnew file mode 100644 index 0000000..a4d56f0 --- /dev/null +++ b/src/connectors/lib/derby.jar diff --git a/src/connectors/lib/derbytools.jar b/src/connectors/lib/derbytools.jar Binary files differnew file mode 100644 index 0000000..216ff3e --- /dev/null +++ b/src/connectors/lib/derbytools.jar diff --git a/src/connectors/lib/mysql-connector-java-5.1.31-bin.jar b/src/connectors/lib/mysql-connector-java-5.1.31-bin.jar Binary files differnew file mode 100644 index 0000000..85ae51d --- /dev/null +++ b/src/connectors/lib/mysql-connector-java-5.1.31-bin.jar diff --git a/src/connectors/lib/ojdbc6-javadoc.jar b/src/connectors/lib/ojdbc6-javadoc.jar Binary files differnew file mode 100644 index 0000000..81dfb08 --- /dev/null +++ b/src/connectors/lib/ojdbc6-javadoc.jar diff --git a/src/connectors/lib/ojdbc6.jar b/src/connectors/lib/ojdbc6.jar Binary files differnew file mode 100644 index 0000000..767eba7 --- /dev/null +++ b/src/connectors/lib/ojdbc6.jar diff --git a/src/connectors/lib/orai18n.jar b/src/connectors/lib/orai18n.jar Binary files differnew file mode 100644 index 0000000..9fad382 --- /dev/null +++ b/src/connectors/lib/orai18n.jar diff --git a/src/connectors/lib/unboundid-ldapsdk-se-javadoc.jar b/src/connectors/lib/unboundid-ldapsdk-se-javadoc.jar Binary files differnew file mode 100644 index 0000000..b724779 --- /dev/null +++ b/src/connectors/lib/unboundid-ldapsdk-se-javadoc.jar diff --git a/src/connectors/lib/unboundid-ldapsdk-se.jar b/src/connectors/lib/unboundid-ldapsdk-se.jar Binary files differnew file mode 100644 index 0000000..0932139 --- /dev/null +++ b/src/connectors/lib/unboundid-ldapsdk-se.jar diff --git a/src/connectors/src/data/io/csv/CSVDataReader.java b/src/connectors/src/data/io/csv/CSVDataReader.java new file mode 100644 index 0000000..6dbc8ff --- /dev/null +++ b/src/connectors/src/data/io/csv/CSVDataReader.java @@ -0,0 +1,248 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ +package data.io.csv; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; + +import data.MVDataEntry; +import data.io.AbstractMVDataReader; + +/** + * Stream-oriented reader from a particular CSV file. + * Always returns lines/items sorted by lexicographical ascending key. + * + * @author lpouzenc + */ +public class CSVDataReader extends AbstractMVDataReader { + + public static final String CSV_DEMO = + //"key,attr,values\n" + + "line3,hello,all;the;others\n" + + "line1,from,csv1;csv1bis\n" + + "line2,hello,all;the;world\n" + + "line1,attr2,csv1\n" + + ",,\n"; + + public static final CSVFormat DEFAULT_CSV_FORMAT = CSVFormat.EXCEL + .withHeader("key","attr","values") + .withIgnoreSurroundingSpaces(true); + + private final CSVFormat format; + private final Reader dataSourceStream; + + private transient MVDataEntry nextEntry; + private transient CSVRecord nextCSVRecord; + private transient Iterator<CSVRecord> csvIt; + + + /** + * Constructs a CSVDataReader object for parsing a CSV input given via dataSourceStream. + * @param dataSourceName A short string representing this reader (for logging) + * @param dataSourceStream A java.io.Reader from which read the actual CSV data, typically a FileReader + * @param alreadySorted If false, memory cost is around 3 times the CSV file size ! + * @param format Specify the exact format used to encode the CSV file (separators, escaping...) + * @throws IOException + */ + public CSVDataReader(String dataSourceName, Reader dataSourceStream, boolean alreadySorted, CSVFormat format) throws IOException { + this.dataSourceName = dataSourceName; + this.format = format; + + if ( alreadySorted ) { + this.dataSourceStream = dataSourceStream; + } else { + BufferedReader bufReader; + if ( dataSourceStream instanceof BufferedReader ) { + bufReader = (BufferedReader) dataSourceStream; + } else { + bufReader = new BufferedReader(dataSourceStream); + } + this.dataSourceStream = readAndSortLines(bufReader); + } + } + + /** + * Constructs a CSVDataReader object with default CSV format (for CSVParser). + * @param dataSourceName A short string representing this reader (for logging) + * @param dataSourceStream A java.io.Reader from which read the actual CSV data, typically a FileReader + * @param alreadySorted If false, memory cost is around 3 times the CSV file size ! + * @throws IOException + */ + public CSVDataReader(String dataSourceName, Reader dataSourceStream, boolean alreadySorted) throws IOException { + this(dataSourceName, dataSourceStream, alreadySorted, DEFAULT_CSV_FORMAT); + } + + /** + * {@inheritDoc} + * Note : multiple iterators on the same instance are not supported + */ + @Override + public Iterator<MVDataEntry> iterator() { + // When a new iterator is requested, everything should be reset + CSVParser parser; + try { + dataSourceStream.reset(); + parser = new CSVParser(dataSourceStream, format); + } catch (IOException e) { + throw new RuntimeException(e); + } + csvIt = parser.iterator(); + nextCSVRecord = null; + nextEntry = null; + return this; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean hasNext() { + if ( nextEntry == null ) { + lookAhead(); + } + return ( nextEntry != null ); + } + + /** + * {@inheritDoc} + */ + @Override + public MVDataEntry next() { + if ( !hasNext() ) { + throw new NoSuchElementException(); + } + // Pop the lookahead record + MVDataEntry res = nextEntry; + nextEntry=null; + // And return it + return res; + } + + /** + * In-memory File sorting, return as a single String + * @param reader + * @return + * @throws IOException + */ + private Reader readAndSortLines(BufferedReader bufReader) throws IOException { + // Put all the CSV in memory, in a SortedSet + SortedSet<String> lineSet = new TreeSet<String>(); + String inputLine; + int totalCSVSize=0; + while ((inputLine = bufReader.readLine()) != null) { + lineSet.add(inputLine); + totalCSVSize += inputLine.length() + 1; + } + bufReader.close(); // Closes also dataSourceStream + + // Put all sorted lines in a String + StringBuilder allLines = new StringBuilder(totalCSVSize); + for ( String line: lineSet) { + allLines.append(line + "\n"); + } + lineSet = null; // Could help the GC if the input file is huge + + // Build a Java Reader from that String + return new StringReader(allLines.toString()); + } + + /** + * A MVDataEntry could be represented on many CSV lines. + * The key is repeated, the attr could change, the values should change (for given key/attr pair) + */ + private void lookAhead() { + MVDataEntry currEntry = null; + + boolean abort=(nextCSVRecord==null && !csvIt.hasNext()); // Nothing to crunch + boolean done=(nextEntry!=null); // Already looked ahead + while (!abort && !done) { + // Try to get a valid CSVRecord + if ( nextCSVRecord == null ) { + nextCSVRecord = nextValidCSVRecord(); + } + // If no more CSV data + if ( nextCSVRecord == null ) { + // Maybe we have a remaining entry to return + if ( currEntry != null ) { + done=true; continue; + } else { + abort=true; continue; + } + } + + // Now we have a valid CSV line to put in a MVDataEntry + String newKey = nextCSVRecord.get("key"); + + + // If no MVDataEntry yet, it's time to create it (we have data to put into) + if ( currEntry == null ) { + currEntry = new MVDataEntry(newKey); + } + // If CSV line key matches MVDataEntry key, appends attr/values on it + // XXX Tricky code : following condition is always true if the previous one is true + if ( currEntry.getKey().equals(newKey) ) { + currEntry.splitAndPut(nextCSVRecord.get("attr"), nextCSVRecord.get("values"), ";"); + nextCSVRecord = null; // Record consumed + } else { + // Keys are different, we are done (and we have remaining CSV data in nextCSVRecord) + done=true; continue; + } + } + + nextEntry = done?currEntry:null; + } + + /** + * Seek for the next valid record in the CSV file + * @return the next valid CSVRecord + */ + private CSVRecord nextValidCSVRecord() { + CSVRecord res = null; + boolean abort = !csvIt.hasNext(); + boolean done = false; + while (!abort && !done) { + // Try to read a CSV line + res = (csvIt.hasNext())?csvIt.next():null; + + // Break if nothing readable + if ( res == null ) { + abort=true; continue; + } + + // Skip invalid and empty lines + String key = res.get("key"); + if ( key != null && ! key.isEmpty() ) { + done=true; continue; + } + } + + return done?res:null; + } +} diff --git a/src/connectors/src/data/io/ldap/LDAPConnectionWrapper.java b/src/connectors/src/data/io/ldap/LDAPConnectionWrapper.java new file mode 100644 index 0000000..3f6497b --- /dev/null +++ b/src/connectors/src/data/io/ldap/LDAPConnectionWrapper.java @@ -0,0 +1,112 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.io.ldap; + +import java.io.Closeable; +import java.io.IOException; + +import com.unboundid.ldap.sdk.BindResult; +import com.unboundid.ldap.sdk.LDAPConnection; +import com.unboundid.ldap.sdk.LDAPConnectionOptions; +import com.unboundid.ldap.sdk.LDAPException; +import com.unboundid.ldap.sdk.ResultCode; + +/** + * TODO javadoc + * + * @author lpouzenc + */ +public class LDAPConnectionWrapper implements Closeable { + + private final LDAPConnection conn; + + /** + * TODO javadoc + * @param host + * @param port + * @param bindDN + * @param password + */ + public LDAPConnectionWrapper(String host, int port, String bindDN, String password) { + LDAPConnectionOptions options = new LDAPConnectionOptions(); + options.setAbandonOnTimeout(true); + options.setAllowConcurrentSocketFactoryUse(true); + options.setAutoReconnect(true); + options.setCaptureConnectStackTrace(true); + options.setConnectTimeoutMillis(2000); // 2 seconds + options.setResponseTimeoutMillis(5000); // 5 seconds + options.setUseSynchronousMode(false); + + BindResult bindResult=null; + try { + conn = new LDAPConnection(options, host, port); + bindResult = conn.bind(bindDN, password); + } + catch (LDAPException e) { + throw new RuntimeException(e); + } + + ResultCode resultCode = bindResult.getResultCode(); + if ( resultCode != ResultCode.SUCCESS ) { + throw new RuntimeException("LDAP Bind failed : " + resultCode); + } + } + + /** + * Builds a new reader against current connection and a LDAP baseDN. + * + * @param dataSourceName Short name of this data source (for logging) + * @param baseDN Search base DN (will return childs of this DN) + * @param keyAttr Attribute name that is the primary key of the entry, identifying the entry in a unique manner + * @param lookAheadAmount Grab this amount of entries at once (in memory-sorted, 128 could be great) + * @return A new reader ready to iterate on search results + */ + public LDAPFlatDataReader newFlatReader(String dataSourceName, String baseDN, String keyAttr, int lookAheadAmount) { + try { + return new LDAPFlatDataReader(dataSourceName, conn, baseDN, keyAttr, lookAheadAmount); + } catch (LDAPException e) { + throw new RuntimeException(e); + } + } + + /** + * Builds a new writer that could insert/update/delete entries on a particular LDAP connection and baseDN. + * + * @param baseDN Search base DN (will return childs of this DN) + * @param keyAttr Attribute name that is the primary key of the entry, identifying the entry in a unique manner + * @return A new writter limited on a particular baseDN + */ + public LDAPFlatDataWriter newFlatWriter(String baseDN, String keyAttr) { + try { + return new LDAPFlatDataWriter(conn, baseDN, keyAttr); + } catch (LDAPException e) { + throw new RuntimeException(e); + } + } + + /** + * Close the current ldap connection. + */ + @Override + public void close() throws IOException { + this.conn.close(); + } +} diff --git a/src/connectors/src/data/io/ldap/LDAPFlatDataReader.java b/src/connectors/src/data/io/ldap/LDAPFlatDataReader.java new file mode 100644 index 0000000..2cc79a8 --- /dev/null +++ b/src/connectors/src/data/io/ldap/LDAPFlatDataReader.java @@ -0,0 +1,178 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.io.ldap; + +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +import com.unboundid.ldap.sdk.Attribute; +import com.unboundid.ldap.sdk.Filter; +import com.unboundid.ldap.sdk.LDAPConnection; +import com.unboundid.ldap.sdk.LDAPException; +import com.unboundid.ldap.sdk.SearchRequest; +import com.unboundid.ldap.sdk.SearchResult; +import com.unboundid.ldap.sdk.SearchResultEntry; +import com.unboundid.ldap.sdk.SearchResultListener; +import com.unboundid.ldap.sdk.SearchResultReference; +import com.unboundid.ldap.sdk.SearchScope; + +import data.MVDataEntry; +import data.io.AbstractMVDataReader; + +/** + * Stream-oriented reader from a particular LDAP connection + * Always returns lines/items sorted by lexicographical ascending key + * Consistent even if there is a Writer on same LDAP connection (useful for sync) + * + * @author lpouzenc + */ +public class LDAPFlatDataReader extends AbstractMVDataReader { + + private final LDAPConnection conn; + private final String baseDN; + private final String keyAttr; + private final int lookAheadAmount; + private final SortedSet<String> keys; + + private transient Iterator<String> keysItCached; + private transient Iterator<String> keysItConsumed; + private transient SortedMap<String, MVDataEntry> entries; + + // Listener to feed LDAP search result in SortedMap without instantiating a big fat SearchResult + private final SearchResultListener keysReqListener = new SearchResultListener() { + private static final long serialVersionUID = 3364745402521913458L; + + @Override + public void searchEntryReturned(SearchResultEntry searchEntry) { + keys.add(searchEntry.getAttributeValue(keyAttr)); + } + + @Override + public void searchReferenceReturned(SearchResultReference searchReference) { + throw new RuntimeException("Unsupported : search request for all '" + keyAttr + "' has returned at least one reference (excepected : an entry)"); + } + }; + + /** + * Construct a new reader that wrap a particular LDAP search on a given connection + * @param dataSourceName Short name of this data source (for logging) + * @param conn Already initialized LDAP connection where run the search + * @param baseDN Search base DN (will return childs of this DN) + * @param keyAttr Attribute name that is the primary key of the entry, identifying the entry in a unique manner + * @param lookAheadAmount Grab this amount of entries at once (in memory-sorted, 128 could be great) + * @throws LDAPException + */ + public LDAPFlatDataReader(String dataSourceName, LDAPConnection conn, String baseDN, String keyAttr, int lookAheadAmount) throws LDAPException { + this.dataSourceName = dataSourceName; + this.conn = conn; + this.baseDN = baseDN; + this.keyAttr = keyAttr; + this.lookAheadAmount = lookAheadAmount; + + // Grab all the entries' keys from LDAP connection and put them in this.keys + this.keys = new TreeSet<String>(); + SearchRequest keysReq = new SearchRequest(keysReqListener, baseDN, SearchScope.ONE, Filter.create("(objectClass=*)"), keyAttr); + conn.search(keysReq); + } + + /** + * {@inheritDoc} + * Note : multiple iterators on the same instance are not supported + */ + @Override + public Iterator<MVDataEntry> iterator() { + // Reset the search (it uses two different iterators on the same set) + keysItCached = keys.iterator(); + keysItConsumed = keys.iterator(); + entries = new TreeMap<String, MVDataEntry>(); + + return this; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean hasNext() { + return (keysItConsumed==null)?false:keysItConsumed.hasNext(); + } + + /** + * {@inheritDoc} + */ + @Override + public MVDataEntry next() { + String wantedKey = keysItConsumed.next(); + + // Feed the lookAhead buffer if it is empty (and there is more elements to grab) + if ( entries.isEmpty() && keysItCached.hasNext() ) { + lookAhead(lookAheadAmount); + } + + //FIXME : it is possible to have inconsistency between "entries" content and keysIt* values if some entry is deleted since we have read all the keys + + // Pop an entry from the lookAhead buffer + MVDataEntry wantedEntry = entries.remove(wantedKey); + if ( wantedEntry == null ) { + throw new NoSuchElementException(); + } + + return wantedEntry; + } + + /** + * Performs look-ahead of amount entries, using the next sorted keys previously queried. + * @param amount + */ + private void lookAhead(int amount) { + if ( amount < 1 ) { + throw new IllegalArgumentException("LookAhead amount has to be >= 1"); + } + try { + // Build a search that matches "amount" next entries + Filter filter = Filter.createEqualityFilter(keyAttr, keysItCached.next()); + for (int i=0; ( i < amount-1 ) && keysItCached.hasNext(); i++) { + filter = Filter.createORFilter(filter, Filter.createEqualityFilter(keyAttr, keysItCached.next())); + } + SearchRequest searchRequest = new SearchRequest(baseDN, SearchScope.ONE, filter, "*"); + + // XXX Could use a second listener, as for the keys + // Get all this entries in memory, convert them in MVDataEntry beans and store them in a SortedMap + SearchResult search = conn.search(searchRequest); + + for (SearchResultEntry ldapEntry: search.getSearchEntries()) { + String key = ldapEntry.getAttributeValue(keyAttr); + MVDataEntry mvEntry = new MVDataEntry(key); + + for ( Attribute attr : ldapEntry.getAttributes() ) { + mvEntry.put(attr.getName(), attr.getValues()); + } + entries.put(key, mvEntry); + } + } catch (LDAPException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/connectors/src/data/io/ldap/LDAPFlatDataWriter.java b/src/connectors/src/data/io/ldap/LDAPFlatDataWriter.java new file mode 100644 index 0000000..d1b8918 --- /dev/null +++ b/src/connectors/src/data/io/ldap/LDAPFlatDataWriter.java @@ -0,0 +1,198 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.io.ldap; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.unboundid.ldap.sdk.Attribute; +import com.unboundid.ldap.sdk.DN; +import com.unboundid.ldap.sdk.DeleteRequest; +import com.unboundid.ldap.sdk.Entry; +import com.unboundid.ldap.sdk.LDAPConnection; +import com.unboundid.ldap.sdk.LDAPException; +import com.unboundid.ldap.sdk.Modification; +import com.unboundid.ldap.sdk.ModificationType; +import com.unboundid.ldap.sdk.ModifyRequest; +import com.unboundid.ldap.sdk.RDN; +import com.unboundid.ldap.sdk.schema.EntryValidator; +import com.unboundid.ldif.LDIFException; + +import data.MVDataEntry; +import data.io.AbstractMVDataWriter; + +/** + * Stream-oriented LDAP writer from a particular LDAP Directory connection. + * + * @author lpouzenc + */ +public class LDAPFlatDataWriter extends AbstractMVDataWriter { + + private final LDAPConnection conn; + private final DN baseDN; + private final String keyAttr; + private final EntryValidator validator; + + /** + * Construct a new writer that could insert/update/delete entries on a particular LDAP connection and baseDN. + * + * @param conn Already initialized LDAP connection where run the search + * @param baseDN Search base DN (will return childs of this DN) + * @param keyAttr Attribute name that is the primary key of the entry, identifying the entry in a unique manner + * @throws LDAPException + */ + public LDAPFlatDataWriter(LDAPConnection conn, String baseDN, String keyAttr) throws LDAPException { + this.conn = conn; + this.baseDN = new DN(baseDN); + this.keyAttr = keyAttr; + this.validator = new EntryValidator(conn.getSchema()); + } + + /** + * {@inheritDoc} + */ + @Override + public void insert(MVDataEntry newEntry) throws LDAPException { + // Build the DN + DN dn = new DN(new RDN(keyAttr, newEntry.getKey()), baseDN); + + // Convert storage objects + Collection<Attribute> attributes = new ArrayList<Attribute>(); + for ( Map.Entry<String, String> entry : newEntry.getAllEntries() ) { + attributes.add(new Attribute(entry.getKey(), entry.getValue())); + } + Entry newLDAPEntry = new Entry(dn, attributes); + + // Add the entry + if ( dryRun ) { + // In dry-run mode, validate the entry + ArrayList<String> invalidReasons = new ArrayList<String>(5); + boolean valid = validator.entryIsValid(newLDAPEntry, invalidReasons); + if ( !valid ) throw new RuntimeException( + "Entry validator has failed to verify this entry :\n" + newLDAPEntry.toLDIFString() + + "Reasons are :\n" + invalidReasons); + } else { + // In real-run mode, insert the entry + try { + conn.add(newLDAPEntry); + } catch (LDAPException e) { + throw new LDAPException(e.getResultCode(), "Error while inserting this entry :\n" + newLDAPEntry.toLDIFString(), e); + } + } + } + + /** + * {@inheritDoc} + */ + @Override + public void update(MVDataEntry updatedEntry, MVDataEntry originalEntry, Set<String> attrToUpdate) throws LDAPException, LDIFException { + // Build the DN + DN dn = new DN(new RDN(keyAttr, updatedEntry.getKey()), baseDN); + + // Convert storage objects + List<Modification> mods = new ArrayList<Modification>(); + for ( String attr : attrToUpdate ) { + Set<String> originalValues = originalEntry.getValues(attr); + Set<String> updatedValues = updatedEntry.getValues(attr); + + Modification modification = null; + + if ( updatedValues.isEmpty() ) { + modification = new Modification(ModificationType.DELETE, attr); + } else { + String[] updatedValuesArr = updatedValues.toArray(new String[0]); + + if ( originalValues.isEmpty() ) { + modification = new Modification(ModificationType.ADD, attr, updatedValuesArr); + } else { + modification = new Modification(ModificationType.REPLACE, attr, updatedValuesArr); + } + } + + mods.add(modification); + } + ModifyRequest modReq = new ModifyRequest(dn, mods); + + // Update the entry + if ( dryRun ) { + // Simulate originalEntry update + Collection<Attribute> attributes = new ArrayList<Attribute>(); + for ( Map.Entry<String, String> entry : originalEntry.getAllEntries() ) { + attributes.add(new Attribute(entry.getKey(), entry.getValue())); + } + Entry originalLDAPEntry = new Entry(dn, attributes); + + // Warning : Unboundid SDK is okay with mandatory attributes with value "" (empty string) + // OpenLDAP do not allow that empty strings in mandatory attributes. + // Empty strings are discarded by MVDataEntry.put() for now. + Entry modifiedLDAPEntry; + try { + modifiedLDAPEntry = Entry.applyModifications(originalLDAPEntry, false, mods); + } catch (LDAPException originalException) { + throw new RuntimeException("Entry update simulation has failed while running applyModifications()\n" + + "original entry : " + originalEntry + "\n" + + "wanted updated entry : " + updatedEntry + "\n" + + "modification request : " + modReq, + originalException); + } + ArrayList<String> invalidReasons = new ArrayList<String>(5); + boolean valid = validator.entryIsValid(modifiedLDAPEntry, invalidReasons); + if ( !valid ) throw new RuntimeException("Entry update simulation has failed while checking entryIsValid()\n" + + "modified entry : " + modifiedLDAPEntry.toLDIFString() + "\n" + + "reasons :" + invalidReasons); + } else { + // In real-run mode, update the entry + try { + conn.modify(modReq); + } catch (LDAPException originalException) { + throw new LDAPException(originalException.getResultCode(), + "Error while updating this entry :\n" + modReq.toLDIFString(), + originalException); + } + } + } + + /** + * {@inheritDoc} + */ + @Override + public void delete(MVDataEntry existingEntry) throws LDAPException { + // Build the DN + DN dn = new DN(new RDN(keyAttr, existingEntry.getKey()), baseDN); + + // Delete the entry + try { + if ( dryRun ) { + //XXX : try to verify the entry existence in dry-run mode ? + } else { + conn.delete(new DeleteRequest(dn)); + } + } catch (LDAPException originalException) { + throw new LDAPException(originalException.getResultCode(), + "Error while deleting this dn : " + dn.toString(), + originalException); + } + } + +} diff --git a/src/connectors/src/data/io/sql/SQLConnectionWrapper.java b/src/connectors/src/data/io/sql/SQLConnectionWrapper.java new file mode 100644 index 0000000..2bab2c8 --- /dev/null +++ b/src/connectors/src/data/io/sql/SQLConnectionWrapper.java @@ -0,0 +1,136 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.io.sql; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.SQLException; + +import data.io.MVDataReader; + +/** + * TODO javadoc + * + * @author lpouzenc + */ +public class SQLConnectionWrapper implements Closeable { + + /** + * Enumeration of supported DBMS. Each use a particular JDBC driver. + */ + public enum DBMSType { oracle, mysql/*, derby*/ } + + private final Connection conn; + + /** + * TODO javadoc + * @param dbms + * @param host + * @param port + * @param ress + * @param user + * @param pass + * @param db + */ + public SQLConnectionWrapper(DBMSType dbms, String host, int port, String ress, String user, String pass, String db) { + + String driverClassName=null; + String url; + + switch ( dbms ) { + case oracle: + driverClassName="oracle.jdbc.driver.OracleDriver"; + url="jdbc:oracle:thin:@" + host + ":" + port + ":" + ress + "/" + db; + break; + case mysql: + driverClassName="com.mysql.jdbc.Driver"; + url="jdbc:mysql://" + host + ":" + port + "/" + db; + break; + /* Could be useful with JUnit tests + case derby: + driverClassName="org.apache.derby.jdbc.EmbeddedDriver"; + url="jdbc:derby:" + db; + break; + */ + default: + throw new IllegalArgumentException("Unsupported DBMSType : " + dbms); + } + + try { + @SuppressWarnings("unchecked") + Class<? extends Driver> clazz = (Class<? extends Driver>) Class.forName(driverClassName); + DriverManager.registerDriver(clazz.newInstance()); + } catch (Exception e) { + throw new RuntimeException("Can't load or register JDBC driver for " + dbms + " (" + driverClassName + ")", e); + } + + try { + conn = DriverManager.getConnection(url, user, pass); + } catch (SQLException e) { + throw new RuntimeException("Can't establish database connection (" + url + ")"); + } + } + + /** + * Builds a new reader against current connection and a File containing a SELECT statement. + * @param name + * @param queryFile + * @return + * @throws IOException + */ + public MVDataReader newReader(String name, File queryFile) throws IOException { + return new SQLRelDataReader(name, conn, queryFile); + } + + /** + * Builds a new reader against current connection and a String containing a SELECT statement. + * @param name + * @param query + * @return + * @throws IOException + */ + public MVDataReader newReader(String name, String query) { + return new SQLRelDataReader(name, conn, query); + } + + /** + * Close the current database connection. + */ + @Override + public void close() throws IOException { + try { + conn.close(); + } catch (SQLException e) { + throw new IOException("Exception occured while trying to close the SQL connection", e); + } + } + + /** + * @return the current database connection (useful for JUnit tests) + */ + public Connection getConn() { + return conn; + } +} diff --git a/src/connectors/src/data/io/sql/SQLRelDataReader.java b/src/connectors/src/data/io/sql/SQLRelDataReader.java new file mode 100644 index 0000000..b6355e9 --- /dev/null +++ b/src/connectors/src/data/io/sql/SQLRelDataReader.java @@ -0,0 +1,173 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.io.sql; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Iterator; + +import data.MVDataEntry; +import data.io.AbstractMVDataReader; + +/** + * Stream-oriented reader from a particular RDBMS source. + * + * @author lpouzenc + */ +public class SQLRelDataReader extends AbstractMVDataReader { + + private final Connection conn; + private final String request; + + private transient String columnNames[]; + private transient ResultSet rs; + private transient boolean didNext; + private transient boolean hasNext; + + /** + * Build a new reader from an existing connection and a File containing a SELECT statement. + * @param dataSourceName A short string representing this reader (for logging) + * @param conn A pre-established SQL data connection + * @param queryFile An SQL file containing an SQL SELECT statement + * @throws IOException + */ + public SQLRelDataReader(String dataSourceName, Connection conn, File queryFile) throws IOException { + this.dataSourceName = dataSourceName; + this.conn = conn; + this.request = readEntireFile(queryFile); + } + + /** + * Build a new reader from an existing connection and a String containing a SELECT statement. + * @param dataSourceName A short string representing this reader (for logging) + * @param conn A pre-established SQL data connection + * @param query A String containing an SQL SELECT statement + * @throws IOException + */ + public SQLRelDataReader(String dataSourceName, Connection conn, String query) { + this.dataSourceName = dataSourceName; + this.conn = conn; + this.request = query; + } + + /** + * {@inheritDoc} + * Note : multiple iterators on the same instance are not supported + */ + @Override + public Iterator<MVDataEntry> iterator() { + try { + // Reset iterator-related attributes + hasNext = false; + didNext = false; + + // Close and free any previous request result + if ( rs != null ) { + rs.close(); + } + // (Re-)Execute the SQL request + Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + rs = stmt.executeQuery(request); + + // Get the column names + ResultSetMetaData rsmd = rs.getMetaData(); + columnNames = new String[rsmd.getColumnCount()]; + for (int i = 0; i < columnNames.length ; i++) { + // Java SQL : all indices starts at 1 (it sucks !) + columnNames[i] = rsmd.getColumnName(i+1); + } + } catch (SQLException e) { + throw new RuntimeException("Could not execute query : " + e.getMessage() + "\n" + request ); + } + + return this; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean hasNext() { + // java.sql.ResultSet don't implement Iterable interface at all + // It's next() don't return anything except hasNext() result but it moves the cursor ! + if (!didNext) { + try { + hasNext = rs.next(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + didNext = true; + } + return hasNext; + } + + /** + * {@inheritDoc} + */ + @Override + public MVDataEntry next() { + MVDataEntry result = null; + try { + if (!didNext) { + rs.next(); + } + didNext = false; + //TODO Instead of always use the first col, user could choose a specific columnName like in LDAP + String key = rs.getString(1); + result = new MVDataEntry(key); + for (int i = 0; i < columnNames.length ; i++) { + // Java SQL : all indices starts at 1 (it sucks !) + result.splitAndPut(columnNames[i], rs.getString(i+1), ";"); // TODO regex should be an option + } + + } catch (SQLException e) { + throw new RuntimeException("Exception while reading next line in SQL resultset", e); + } + + return result; + } + + /** + * Helper function to load and entire file as a String. + * @param file + * @return + * @throws IOException + */ + private static String readEntireFile(File file) throws IOException { + FileReader input = new FileReader(file); + StringBuilder contents = new StringBuilder(); + char[] buffer = new char[4096]; + int read = 0; + do { + contents.append(buffer, 0, read); + read = input.read(buffer); + } while (read >= 0); + input.close(); + + return contents.toString(); + } +} diff --git a/src/core/.classpath b/src/core/.classpath new file mode 100644 index 0000000..f7de406 --- /dev/null +++ b/src/core/.classpath @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="UTF-8"?> +<classpath> + <classpathentry kind="src" path="src"/> + <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/> + <classpathentry kind="src" path="JUTests"/> + <classpathentry kind="lib" path="lib/guava-16.0.1.jar"/> + <classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/> + <classpathentry kind="output" path="bin"/> +</classpath> diff --git a/src/core/.project b/src/core/.project new file mode 100644 index 0000000..acda864 --- /dev/null +++ b/src/core/.project @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="UTF-8"?> +<projectDescription> + <name>SSSync_Core</name> + <comment></comment> + <projects> + </projects> + <buildSpec> + <buildCommand> + <name>org.eclipse.jdt.core.javabuilder</name> + <arguments> + </arguments> + </buildCommand> + </buildSpec> + <natures> + <nature>org.eclipse.jdt.core.javanature</nature> + </natures> +</projectDescription> diff --git a/src/core/.settings/org.eclipse.jdt.core.prefs b/src/core/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..8000cd6 --- /dev/null +++ b/src/core/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,11 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=1.6 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.source=1.6 diff --git a/src/core/JUTests/data/MVDataEntryTest.java b/src/core/JUTests/data/MVDataEntryTest.java new file mode 100644 index 0000000..19ccb46 --- /dev/null +++ b/src/core/JUTests/data/MVDataEntryTest.java @@ -0,0 +1,93 @@ +package data; + +import static org.junit.Assert.*; + +import org.junit.Test; + +import com.google.common.collect.HashMultimap; + +public class MVDataEntryTest { + + @Test + public void testMVDataEntryStringIntInt() { + String expected = "{key=line1, attrValPairs={k4=[v4], k1=[v1b, v1a, v1c], k2=[v1c]}}"; + + MVDataEntry e1 = new MVDataEntry("line1", 1, 1); + HashMultimap<String, String> e1v = e1.getAttrValPairs(); + e1v.put("k1", "v1a"); + e1v.put("k1", "v1b"); + e1v.put("k1", "v1b"); // Twice, should disappear silently + e1v.put("k1", "v1c"); + e1v.put("k2", "v1c"); + e1v.put("k4", "v4"); + + assertEquals(expected, e1.toString()); + + } + + @Test + public void testMerge() { + // Test data + MVDataEntry e1 = new MVDataEntry("10"); + HashMultimap<String, String> e1v = e1.getAttrValPairs(); + e1v.put("k1", "v1a"); + e1v.put("k1", "v1b"); + e1v.put("k1", "v1c"); + e1v.put("k2", "v2"); + e1v.put("k4", "v4"); + + MVDataEntry e2 = new MVDataEntry("2"); + HashMultimap<String, String> e2v = e2.getAttrValPairs(); + e2v.put("k2", "v2"); + e2v.put("k1", "v1b"); + e2v.put("k3", "v3"); + + MVDataEntry r1 = new MVDataEntry(e1); + r1.mergeValues(true, e2); + assertNotSame(r1, e1); + String expected1 = "{key=10, attrValPairs={k3=[v3], k4=[v4], k1=[v1b, v1a, v1c], k2=[v2]}}"; + assertEquals(expected1, r1.toString()); + + MVDataEntry r2 = new MVDataEntry(e2); + r2.mergeValues(true, e1); + assertNotSame(r2, e2); + String expected2 = "{key=2, attrValPairs={k3=[v3], k4=[v4], k1=[v1b, v1a, v1c], k2=[v2]}}"; + assertEquals(expected2, r2.toString()); + + MVDataEntry r3 = new MVDataEntry(e1); + r3.mergeValues(false, e2); + assertNotSame(r3, e1); + String expected3 = "{key=10, attrValPairs={k3=[v3], k4=[v4], k1=[v1b], k2=[v2]}}"; + //System.out.println(expected3); + //System.out.println(r3.toString()); + assertEquals(expected3, r3.toString()); + + MVDataEntry r4 = new MVDataEntry(e2); + r4.mergeValues(false, e1); + assertNotSame(r4, e1); + String expected4 = "{key=2, attrValPairs={k3=[v3], k4=[v4], k1=[v1b, v1a, v1c], k2=[v2]}}"; + assertEquals(expected4, r4.toString()); + + assertTrue(!r2.equals(r3)); + assertEquals(r2,r4); + } + + @Test + public void testSplitAndPut() { + MVDataEntry r1 = new MVDataEntry("10"); + r1.splitAndPut("k1", "v1a;v1b;v1c", ";"); + r1.put("k2", "v2", null); // splitAndPut does not support null regex anymore, use put() + r1.splitAndPut("k4", "v4", "^$"); + + MVDataEntry expected1 = new MVDataEntry("10"); + HashMultimap<String, String> expected1v = expected1.getAttrValPairs(); + expected1v.put("k1", "v1a"); + expected1v.put("k1", "v1b"); + expected1v.put("k1", "v1c"); + expected1v.put("k2", "v2"); + expected1v.put("k4", "v4"); + + assertEquals(r1,expected1); + } + +} diff --git a/src/core/JUTests/data/io/filters/MVDataCombinerTest.java b/src/core/JUTests/data/io/filters/MVDataCombinerTest.java new file mode 100644 index 0000000..5d32dd8 --- /dev/null +++ b/src/core/JUTests/data/io/filters/MVDataCombinerTest.java @@ -0,0 +1,148 @@ +package data.io.filters; + +import static org.junit.Assert.*; + +import java.util.Iterator; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import data.MVDataEntry; +import data.filters.MVDataCombiner; +import data.filters.MVDataCombiner.MVDataCombineMode; +import data.io.MVDataReader; +import data.io.stub.StubDataReader; + +public class MVDataCombinerTest { + + @Rule + public ExpectedException exception = ExpectedException.none(); + + @Test + public void testOutOfOrderCase() { + // Test Data + MVDataEntry e10 = new MVDataEntry("line2"); + e10.getAttrValPairs().put("merge", "e10"); + MVDataEntry e11 = new MVDataEntry("line1"); + e11.getAttrValPairs().put("merge", "e11"); + + MVDataEntry e21 = new MVDataEntry("line2"); + e21.getAttrValPairs().put("merge", "e21"); + + MVDataEntry[][] fakeEntries = new MVDataEntry[][] { + new MVDataEntry[] { e10, e11 }, + new MVDataEntry[] { e21 }, + }; + + MVDataCombineMode mergeModes[] = new MVDataCombineMode[]{ + MVDataCombineMode.PRIMARY_SOURCE, + MVDataCombineMode.MERGE_APPEND, + }; + + // Expected results + MVDataEntry line1 = new MVDataEntry(e10); + line1.mergeValues(true, e21); + + MVDataEntry expected[] = new MVDataEntry[] { + line1, + null /* Should throw UnsupportedOperationException() before comparing */ + }; + + // Test run + exception.expect(UnsupportedOperationException.class); + doCombineTest(expected, fakeEntries, mergeModes); + } + + + @Test + public void testGeneralCase() { + + // Test Data + MVDataEntry e10 = new MVDataEntry("line3"); + e10.getAttrValPairs().put("from1", "e10"); + e10.getAttrValPairs().put("merge", "e10"); + MVDataEntry e11 = new MVDataEntry("line4"); + e11.getAttrValPairs().put("from1", "e11"); + e11.getAttrValPairs().put("merge", "e11"); + + MVDataEntry e20 = new MVDataEntry("line1"); + e20.getAttrValPairs().put("from2", "e20"); + e20.getAttrValPairs().put("merge", "e20"); + MVDataEntry e21 = new MVDataEntry("line2"); + e21.getAttrValPairs().put("from2", "e21"); + e21.getAttrValPairs().put("merge", "e21"); + MVDataEntry e22 = new MVDataEntry("line3"); + e22.getAttrValPairs().put("from2", "e22"); + e22.getAttrValPairs().put("merge", "e22"); + + MVDataEntry e30 = new MVDataEntry("line2"); + e30.getAttrValPairs().put("from3", "e30"); + e30.getAttrValPairs().put("merge", "e30"); + + + MVDataEntry[][] fakeEntries = new MVDataEntry[][] { + new MVDataEntry[] { e10, e11 }, + new MVDataEntry[] { e20, e21, e22 }, + new MVDataEntry[] { e30 }, + }; + + MVDataCombineMode mergeModes[] = new MVDataCombineMode[]{ + MVDataCombineMode.PRIMARY_SOURCE, + MVDataCombineMode.MERGE_REPLACE, + MVDataCombineMode.MERGE_APPEND, + }; + + // Expected results + MVDataEntry line1 = new MVDataEntry(e20); + + MVDataEntry line2 = new MVDataEntry(e21); + line2.mergeValues(true, e30); + + MVDataEntry line3 = new MVDataEntry(e10); + line3.mergeValues(false, e22); + + MVDataEntry line4 = new MVDataEntry(e11); + + MVDataEntry expected[] = new MVDataEntry[] { + line1,line2,line3,line4 + }; + + // Test run + doCombineTest(expected, fakeEntries, mergeModes); + } + + // TODO : test all Combine modes + + /** + * Helper function to factorise Combiner tests. + * @param expected + * @param fakeEntries + * @param mergeModes + */ + public void doCombineTest(MVDataEntry expected[], MVDataEntry[][] fakeEntries, MVDataCombineMode mergeModes[]) { + // Test init + MVDataReader readers[] = new MVDataReader[fakeEntries.length]; + for (int i = 0; i < fakeEntries.length; i++) { + readers[i] = new StubDataReader("fakeReader"+i,fakeEntries[i]); + } + + MVDataCombiner combiner = new MVDataCombiner("combiner", readers, mergeModes); + + // Test twice to check if asking a new iterator "rewinds" correctly + for (int i=0;i<2;i++) { + //System.out.println("Loop " + (i+1)); + + Iterator<MVDataEntry> combinerIt = combiner.iterator(); + for (int j = 0; j < expected.length; j++) { + assertTrue(combinerIt.hasNext()); + MVDataEntry item = combinerIt.next(); + //System.out.println(expected[i]); + //System.out.println(item); + //System.out.println(); + assertEquals(expected[j], item); + } + assertFalse(combinerIt.hasNext()); + } + } +} diff --git a/src/core/build.xml b/src/core/build.xml new file mode 100644 index 0000000..e46c220 --- /dev/null +++ b/src/core/build.xml @@ -0,0 +1,78 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- WARNING: Eclipse auto-generated file. + Any modifications will be overwritten. + To include a user specific buildfile here, simply create one in the same + directory with the processing instruction <?eclipse.ant.import?> + as the first entry and export the buildfile again. --> +<project basedir="." default="build" name="SSSync_Core"> + <property environment="env"/> + <property name="SSSync_Connectors.location" value="../connectors"/> + <property name="SSSync_Main.location" value="../main"/> + <property name="ECLIPSE_HOME" value="../../../../../../usr/lib/eclipse"/> + <property name="debuglevel" value="source,lines,vars"/> + <property name="target" value="1.6"/> + <property name="source" value="1.6"/> + <path id="JUnit 4.libraryclasspath"> + <pathelement location="../../../../../../usr/share/eclipse/dropins/jdt/plugins/org.junit_4.8.2.dist/junit.jar"/> + <pathelement location="../../../../../../usr/share/eclipse/dropins/jdt/plugins/org.hamcrest.core_1.1.0.jar"/> + </path> + <path id="SSSync_Core.classpath"> + <pathelement location="bin"/> + <pathelement location="lib/guava-16.0.1.jar"/> + <path refid="JUnit 4.libraryclasspath"/> + </path> + <target name="init"> + <mkdir dir="bin"/> + <copy includeemptydirs="false" todir="bin"> + <fileset dir="src"> + <exclude name="**/*.java"/> + </fileset> + </copy> + <copy includeemptydirs="false" todir="bin"> + <fileset dir="JUTests"> + <exclude name="**/*.java"/> + </fileset> + </copy> + </target> + <target name="clean"> + <delete dir="bin"/> + </target> + <target depends="clean" name="cleanall"/> + <target depends="build-subprojects,build-project" name="build"/> + <target name="build-subprojects"/> + <target depends="init" name="build-project"> + <echo message="${ant.project.name}: ${ant.file}"/> + <javac debug="true" debuglevel="${debuglevel}" destdir="bin" includeantruntime="false" source="${source}" target="${target}"> + <src path="src"/> + <src path="JUTests"/> + <classpath refid="SSSync_Core.classpath"/> + </javac> + </target> + <target description="Build all projects which reference this project. Useful to propagate changes." name="build-refprojects"> + <ant antfile="build.xml" dir="${SSSync_Connectors.location}" inheritAll="false" target="clean"/> + <ant antfile="build.xml" dir="${SSSync_Connectors.location}" inheritAll="false" target="build"> + <propertyset> + <propertyref name="build.compiler"/> + </propertyset> + </ant> + <ant antfile="build.xml" dir="${SSSync_Main.location}" inheritAll="false" target="clean"/> + <ant antfile="build.xml" dir="${SSSync_Main.location}" inheritAll="false" target="build"> + <propertyset> + <propertyref name="build.compiler"/> + </propertyset> + </ant> + </target> + <target description="copy Eclipse compiler jars to ant lib directory" name="init-eclipse-compiler"> + <copy todir="${ant.library.dir}"> + <fileset dir="${ECLIPSE_HOME}/plugins" includes="org.eclipse.jdt.core_*.jar"/> + </copy> + <unzip dest="${ant.library.dir}"> + <patternset includes="jdtCompilerAdapter.jar"/> + <fileset dir="${ECLIPSE_HOME}/plugins" includes="org.eclipse.jdt.core_*.jar"/> + </unzip> + </target> + <target description="compile project with Eclipse compiler" name="build-eclipse-compiler"> + <property name="build.compiler" value="org.eclipse.jdt.core.JDTCompilerAdapter"/> + <antcall target="build"/> + </target> +</project> diff --git a/src/core/lib/guava-16.0.1.jar b/src/core/lib/guava-16.0.1.jar Binary files differnew file mode 100644 index 0000000..2c8127d --- /dev/null +++ b/src/core/lib/guava-16.0.1.jar diff --git a/src/core/src/data/MVDataEntry.java b/src/core/src/data/MVDataEntry.java new file mode 100644 index 0000000..f92a141 --- /dev/null +++ b/src/core/src/data/MVDataEntry.java @@ -0,0 +1,238 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data; + +import java.util.HashSet; +import java.util.Map.Entry; +import java.util.Set; + +import com.google.common.collect.HashMultimap; + +/** + * Generic Multi-Valued data type. Each object store a particular entry. + * Semantics are like in LDAP directories : an entry = a key + a set of multi-valued attributes. + * Relational data like in RDMS are more constrained : columns are fixed for an entire table. + * Null and empty string attribute value are silently discarded. + * + * @author lpouzenc + */ +public class MVDataEntry implements Comparable<MVDataEntry> { + + /** + * The key part that identify this particular entry. + */ + private final String key; + /** + * The data part of this particular entry. + */ + private HashMultimap<String,String> attrValPairs; + + // XXX : add an HashMap for meta or constraints ? + + // Constructors + + /** + * Build a fresh empty MVDataEntry. + * @param key Unique key identifying this entry + */ + public MVDataEntry(String key) { + if ( key == null ) { + throw new IllegalArgumentException("key must be non-null"); + } + this.key = key; + this.attrValPairs = HashMultimap.create(); + } + + /** + * Build a fresh empty MVDataEntry with hints about expected attr/values count. + * @param key Unique key identifying this entry + */ + public MVDataEntry(String key, int expectedAttribs, int expectedValuesPerAttrib) { + if ( key == null ) { + throw new IllegalArgumentException("key must be non-null"); + } + this.key = key; + this.attrValPairs = HashMultimap.create(expectedAttribs, expectedValuesPerAttrib); + } + + /** + * Deep copy of an existing MVDataEntry. + * @param key Unique key identifying this entry + */ + public MVDataEntry(final MVDataEntry copyFrom) { + this.key=copyFrom.key; // String is immutable, so ref copy is okay + this.attrValPairs = HashMultimap.create(copyFrom.attrValPairs); + } + + /** + * Proxy function to return all attribute/value pairs. + * One can use read a MVDataEntry without depending on non-standard HashMultimap. + * @return + */ + public Set<Entry<String, String>> getAllEntries() { + return this.attrValPairs.entries(); + } + + /** + * Proxy function to add an attribute/value pair in attrValPairs. + * One can use MVDataEntry without depending on non-standard HashMultimap. + * + * @param attr + * @param value + */ + public void put(String attr, String... values) { + for (String value: values) { + if ( value != null && !value.isEmpty() ) { + this.attrValPairs.put(attr, value); + } + } + } + + /** + * Proxy function to get all values from a particular attribute. + * One can use MVDataEntry without depending on non-standard HashMultimap. + * @param attr + * @return + */ + public Set<String> getValues(String attr) { + return this.attrValPairs.get(attr); + } + + /** + * Helper function to insert multiple values from a single string. + * + * @param attr + * @param value + * @param splitRegex + */ + public void splitAndPut(String attr, String value, String splitRegex) { + if ( value != null ) { + for (String v : value.split(splitRegex)) { + put(attr, v); + } + } + } + + /** + * Helper function to return list of changed attributes. + * Note : this don't keep track of deleted attributes. + * @param original + * @return + */ + public Set<String> getChangedAttributes(MVDataEntry original) { + HashSet<String> result = new HashSet<String>(); + + for (String attr: this.attrValPairs.keySet()) { + Set<String> thisValue = this.attrValPairs.get(attr); + Set<String> originalValue = original.attrValPairs.get(attr); + if ( ! thisValue.equals(originalValue) ) { + result.add(attr); + } + } + + return result; + } + + /** + * Augment this entry with attr/values from other entries. + * @param appendMode Select behavior on an existing attribute : append values or replace them + * @param entries Entries to merge with current entry + */ + public void mergeValues(boolean appendMode, MVDataEntry... entries) { + for(MVDataEntry entry : entries) { + if ( ! appendMode ) { + for (String attr : entry.attrValPairs.keySet()) { + this.attrValPairs.removeAll(attr); + } + } + this.attrValPairs.putAll(entry.attrValPairs); + } + } + + /** + * Check if this entry seems contains useful data. + * @return true if this entry seems contains useful data + */ + public boolean isValid() { + boolean validKey=(this.key != null && this.key.length() > 0 ); + boolean validVal=(this.attrValPairs != null && ! this.attrValPairs.isEmpty()); + + return (validKey && validVal); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean equals(Object obj) { + // Check for self-comparison (compare object references) + if ( this == obj ) { return true; } + // Check non-nullity and type + if ( !( obj instanceof MVDataEntry) ) { return false; } + // Cast safely + MVDataEntry other = (MVDataEntry) obj; + // Check all fields (known to be always non null) + return ( this.key.equals(other.key) && this.attrValPairs.equals(other.attrValPairs) ); + } + + /** + * Compares entries. Ordering of entries is the ordering of their keys. + * (java.lang.String default ordering : lexicographical ascending order) + */ + @Override + public int compareTo(MVDataEntry other) { + return this.key.compareTo(other.key); + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() { + return "{key=" + key + ", attrValPairs=" + attrValPairs.toString() + "}"; + } + + + // Boring accessors + /** + * @return the attrValPairs + */ + public HashMultimap<String, String> getAttrValPairs() { + return attrValPairs; + } + + /** + * @param attrValPairs the attrValPairs to set + */ + public void setAttrValPairs(HashMultimap<String, String> attrValPairs) { + this.attrValPairs = attrValPairs; + } + + /** + * @return the key (guaranteed to be non-null) + */ + public String getKey() { + return key; + } + + + +} diff --git a/src/core/src/data/filters/MVDataCombiner.java b/src/core/src/data/filters/MVDataCombiner.java new file mode 100644 index 0000000..1b2eb3f --- /dev/null +++ b/src/core/src/data/filters/MVDataCombiner.java @@ -0,0 +1,164 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.filters; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +import data.MVDataEntry; +import data.io.AbstractMVDataReader; +import data.io.MVDataReader; + +/** + * Combines arbitrary number of MVData* sources while behaving same as AbstractMVDataReader. + * This could enable a sync implementation to merge multiple sources + * before sync'ing in a transparent manner. + * To prevent memory consumption, this assumes that all sources will be read + * with lexicographical ascending order on the "key" field. + * + * @author lpouzenc + */ +public class MVDataCombiner extends AbstractMVDataReader { + + public enum MVDataCombineMode { PRIMARY_SOURCE, MERGE_APPEND, MERGE_REPLACE, OVERRIDE }; + + private final MVDataReader[] readers; + private final MVDataCombineMode[] mergeModes; + + private transient Iterator<MVDataEntry>[] readerIterators; + private transient MVDataEntry[] lookAheadData; + private transient String lastKey; + + + public MVDataCombiner(String dataSourceName, MVDataReader[] readers, MVDataCombineMode mergeModes[]) { + if ( readers == null || mergeModes == null || (mergeModes.length != readers.length) ) { + throw new IllegalArgumentException("readers and mergeModes arrays should have same size"); + } + if ( ! (mergeModes.length > 0) || mergeModes[0] != MVDataCombineMode.PRIMARY_SOURCE ) { + throw new IllegalArgumentException("MVDataCombiner first mergeModes should always be PRIMARY_SOURCE"); + } + + this.dataSourceName = dataSourceName; + this.readers = readers.clone(); + this.mergeModes = mergeModes.clone(); + } + + /** + * {@inheritDoc} + */ + @Override + @SuppressWarnings("unchecked") /* for new Iterator[...] */ + public Iterator<MVDataEntry> iterator() { + // Be cautious to reset everything + readerIterators = new Iterator[readers.length]; + for (int i=0; i<readers.length;i++) { + readerIterators[i] = readers[i].iterator(); + } + lookAheadData = new MVDataEntry[readers.length]; + lastKey = null; + + return this; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean hasNext() { + for ( MVDataEntry line : lookAheadData ) { + if ( line != null ) { return true; } + } + for ( MVDataReader reader : readers ) { + if ( reader.hasNext() ) { return true; } + } + return false; + } + + /** + * {@inheritDoc} + */ + @Override + public MVDataEntry next() { + + final String currentKey = lookAheadAll(); + + // Check if there was unsorted lines in source data + if ( lastKey != null && (currentKey.compareTo(lastKey) < 0) ) { + //XXX : this is checked here and in SafeDataReader (redundant), but both are optionnal... + throw new UnsupportedOperationException("At least one data source is out of order. " + + "Data sources are excepted to be read sorted by MVDataEntry key (ascending lexicogrpahical order)"); + } + + // Merge all data sources for key currentKey + MVDataEntry result = null; + for ( int i=0; i<lookAheadData.length; i++) { + if ( lookAheadData[i] != null && lookAheadData[i].getKey().equals(currentKey) ) { + if ( result == null ) { + result = lookAheadData[i]; + } else { + //XXX : some items in LDAP could have constrains like : "not multi-valued". Force MERGE_REPLACE mode ? + //FIXME : honor all Combine modes + result.mergeValues( (mergeModes[i] == MVDataCombineMode.MERGE_APPEND ),lookAheadData[i]); + } + lookAheadData[i]=null; // "Pop" the used entry + } + } + + lastKey = currentKey; + + return result; + } + + private String lookAheadAll() { + String minKey=null; + + // Feed the look-ahead buffer (look forward by 1 value for each reader) + for ( int i=0; i<lookAheadData.length; i++) { + if ( lookAheadData[i] == null && readerIterators[i].hasNext() ) { + lookAheadData[i] = readerIterators[i].next(); + } + } + + // Find the least RelData key from look-ahead buffers + for (MVDataEntry entry: lookAheadData) { + if ( entry != null ) { + final String minKeyCandidate = entry.getKey(); + if ( minKey == null || minKey.compareTo(minKeyCandidate) > 0 ) { + minKey = minKeyCandidate; + } + } + } + + // Sanity checks + if ( minKey == null ) { + // Every reader is empty and look-ahead buffer is empty (hasNext() should have said false) + throw new NoSuchElementException(); + } + + return minKey; + } + + // Boring accessors + + public String getLastKey() { + return lastKey; + } +} diff --git a/src/core/src/data/io/AbstractMVDataReader.java b/src/core/src/data/io/AbstractMVDataReader.java new file mode 100644 index 0000000..3e63de1 --- /dev/null +++ b/src/core/src/data/io/AbstractMVDataReader.java @@ -0,0 +1,49 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.io; + +/** + * Stream-oriented abstract reader from a particular data source. + * Memory footprint should not depends on readable line count nor next() call count. + * + * @author lpouzenc + */ +public abstract class AbstractMVDataReader implements MVDataReader { + + protected String dataSourceName="(unknown source)"; + + /** + * Not supported (Readers are read-only). + */ + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + /* (non-Javadoc) + * @see data.io.MVDataReader#getDataSourceName() + */ + @Override + public String getDataSourceName() { + return dataSourceName; + } + +} diff --git a/src/core/src/data/io/AbstractMVDataWriter.java b/src/core/src/data/io/AbstractMVDataWriter.java new file mode 100644 index 0000000..454e8ce --- /dev/null +++ b/src/core/src/data/io/AbstractMVDataWriter.java @@ -0,0 +1,70 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.io; + +import java.util.Set; + +import data.MVDataEntry; + +/** + * Stream-oriented abstract writer from a particular data source. + * All derived writers should honor a dry-run mode. + * + * @author lpouzenc + */ +public abstract class AbstractMVDataWriter implements MVDataWriter { + //TODO : not so useful. Interface extraction was not a good idea ? + + /** + * Dry-run mode flag (disabled by default) + */ + protected boolean dryRun=false; + + /* (non-Javadoc) + * @see data.io.MVDataWriter#isDryRun() + */ + public boolean isDryRun() { + return dryRun; + } + + /* (non-Javadoc) + * @see data.io.MVDataWriter#setDryRun(boolean) + */ + public void setDryRun(boolean dryRun) { + this.dryRun = dryRun; + } + + /* (non-Javadoc) + * @see data.io.MVDataWriter#insert(data.MVDataEntry) + */ + @Override + public abstract void insert(MVDataEntry newEntry) throws Exception; + /* (non-Javadoc) + * @see data.io.MVDataWriter#update(data.MVDataEntry, data.MVDataEntry, java.util.Set) + */ + @Override + public abstract void update(MVDataEntry updatedEntry, MVDataEntry originalEntry, Set<String> attrToUpdate) throws Exception; + /* (non-Javadoc) + * @see data.io.MVDataWriter#delete(data.MVDataEntry) + */ + @Override + public abstract void delete(MVDataEntry existingEntry) throws Exception; +} diff --git a/src/core/src/data/io/MVDataReader.java b/src/core/src/data/io/MVDataReader.java new file mode 100644 index 0000000..8a9871a --- /dev/null +++ b/src/core/src/data/io/MVDataReader.java @@ -0,0 +1,39 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.io; + +import java.util.Iterator; + +import data.MVDataEntry; + +/** + * TODO javadoc + * + * @author lpouzenc + */ +public interface MVDataReader extends Iterator<MVDataEntry>, Iterable<MVDataEntry>{ + + /** + * @return the dataSourceName + */ + public String getDataSourceName(); + +}
\ No newline at end of file diff --git a/src/core/src/data/io/MVDataWriter.java b/src/core/src/data/io/MVDataWriter.java new file mode 100644 index 0000000..2f16fbc --- /dev/null +++ b/src/core/src/data/io/MVDataWriter.java @@ -0,0 +1,45 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.io; + +import java.util.Set; + +import data.MVDataEntry; + +/** + * TODO javadoc + * + * @author lpouzenc + */ +public interface MVDataWriter { + + public boolean isDryRun(); + public void setDryRun(boolean dryRun); + + public void insert(MVDataEntry newEntry) throws Exception; + + public void update(MVDataEntry updatedEntry, + MVDataEntry originalEntry, Set<String> attrToUpdate) + throws Exception; + + public void delete(MVDataEntry existingEntry) throws Exception; + +}
\ No newline at end of file diff --git a/src/core/src/data/io/stub/StubDataReader.java b/src/core/src/data/io/stub/StubDataReader.java new file mode 100644 index 0000000..ed91267 --- /dev/null +++ b/src/core/src/data/io/stub/StubDataReader.java @@ -0,0 +1,63 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.io.stub; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +import data.MVDataEntry; +import data.io.AbstractMVDataReader; + +/** + * Stub reader implementation for automated tests. + * + * @author lpouzenc + */ +public class StubDataReader extends AbstractMVDataReader { + + private final MVDataEntry fakeEntries[]; + private int cursorRead; + + public StubDataReader(String dataSourceName, MVDataEntry[] fakeEntries) { + this.dataSourceName = dataSourceName; + this.fakeEntries = fakeEntries.clone(); + } + + @Override + public Iterator<MVDataEntry> iterator() { + this.cursorRead = 0; + return this; + } + + @Override + public boolean hasNext() { + return cursorRead < fakeEntries.length; + } + + @Override + public MVDataEntry next() { + if ( ! hasNext() ) { + throw new NoSuchElementException(); + } + return fakeEntries[cursorRead++]; + } + +} diff --git a/src/core/src/data/io/stub/StubDataWriter.java b/src/core/src/data/io/stub/StubDataWriter.java new file mode 100644 index 0000000..cd08e77 --- /dev/null +++ b/src/core/src/data/io/stub/StubDataWriter.java @@ -0,0 +1,104 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.io.stub; + +import java.util.Set; + +import data.MVDataEntry; +import data.io.AbstractMVDataWriter; + +/** + * Stub writer implementation for automated tests. + * + * @author lpouzenc + */ +public class StubDataWriter extends AbstractMVDataWriter { + + enum OpKind { INSERT, UPDATE, DELETE }; + + private final int maxLogEntries; + + private OpKind opLog[]; + private MVDataEntry opData[]; + private int cursorLog; + + public StubDataWriter(int maxLogEntries) { + this.maxLogEntries = maxLogEntries; + this.opLog = new OpKind[maxLogEntries]; + this.opData = new MVDataEntry[maxLogEntries]; + } + + @Override + public void insert(MVDataEntry newline) { + if ( cursorLog >= maxLogEntries) { + throw new IllegalStateException(); + } + opLog[cursorLog]=OpKind.INSERT; + opData[cursorLog]=newline; + cursorLog++; + } + + @Override + public void update(MVDataEntry updatedLine, MVDataEntry originalLine, Set<String> attrToUpdate) { + if ( cursorLog >= maxLogEntries) { + throw new IllegalStateException(); + } + opLog[cursorLog]=OpKind.UPDATE; + opData[cursorLog]=updatedLine; + cursorLog++; + } + + @Override + public void delete(MVDataEntry existingLine) { + if ( cursorLog >= maxLogEntries) { + throw new IllegalStateException(); + } + opLog[cursorLog]=OpKind.DELETE; + opData[cursorLog]=existingLine; + cursorLog++; + } + + @Override + public String toString() { + StringBuffer buf = new StringBuffer(); + + for (int i = 0; i < cursorLog; i++) { + buf.append(opLog[i] + ": " + opData[i] + "\n"); + } + + return buf.toString(); + } + + /** + * @return the opLog + */ + public OpKind[] getOpLog() { + return opLog.clone(); + } + + /** + * @return the opData + */ + public MVDataEntry[] getOpData() { + return opData.clone(); + } + +} diff --git a/src/core/src/sync/AbstractSyncTask.java b/src/core/src/sync/AbstractSyncTask.java new file mode 100644 index 0000000..e2ae94d --- /dev/null +++ b/src/core/src/sync/AbstractSyncTask.java @@ -0,0 +1,71 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package sync; + +import java.util.concurrent.Callable; + +/** + * Abstract class to define a common base of all kind of synchronization algorithms. + * + * @author lpouzenc + */ +public abstract class AbstractSyncTask implements Callable<Boolean> { + + /** + * Pretty task name to be inserted in log lines + */ + protected String taskName="(unknown task)"; + + /** + * Dry-run mode flag (disabled by default) + */ + protected boolean dryRun=false; + + /** + * Main method that do the actual sync + */ + public abstract Boolean call(); + + + // Boring accessors + + /** + * @return the dryRun mode status (enabled/disabled) + */ + public boolean isDryRun() { + return dryRun; + } + + /** + * @param dryRun the dryRun mode to set (enabled/disabled) + */ + public void setDryRun(boolean dryRun) { + this.dryRun = dryRun; + } + + /** + * @return the taskName + */ + public String getTaskName() { + return taskName; + } + +} diff --git a/src/main/.classpath b/src/main/.classpath new file mode 100644 index 0000000..33bcbdb --- /dev/null +++ b/src/main/.classpath @@ -0,0 +1,16 @@ +<?xml version="1.0" encoding="UTF-8"?> +<classpath> + <classpathentry kind="src" path="src"/> + <classpathentry kind="src" path="JUTests"/> + <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/> + <classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/> + <classpathentry combineaccessrules="false" kind="src" path="/SSSync_Core"/> + <classpathentry kind="lib" path="lib/log4j-1.2.17.jar"/> + <classpathentry kind="lib" path="lib/snakeyaml-1.11.jar"> + <attributes> + <attribute name="javadoc_location" value="jar:platform:/resource/SSSync/lib/snakeyaml-1.11-javadoc.jar!/"/> + </attributes> + </classpathentry> + <classpathentry combineaccessrules="false" kind="src" path="/SSSync_Connectors"/> + <classpathentry kind="output" path="bin"/> +</classpath> diff --git a/src/main/.project b/src/main/.project new file mode 100644 index 0000000..33a3a78 --- /dev/null +++ b/src/main/.project @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="UTF-8"?> +<projectDescription> + <name>SSSync_Main</name> + <comment></comment> + <projects> + </projects> + <buildSpec> + <buildCommand> + <name>org.eclipse.jdt.core.javabuilder</name> + <arguments> + </arguments> + </buildCommand> + </buildSpec> + <natures> + <nature>org.eclipse.jdt.core.javanature</nature> + </natures> +</projectDescription> diff --git a/src/main/.settings/org.eclipse.jdt.core.prefs b/src/main/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..8000cd6 --- /dev/null +++ b/src/main/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,11 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=1.6 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.source=1.6 diff --git a/src/main/JUTests/AllClientServerTests.java b/src/main/JUTests/AllClientServerTests.java new file mode 100644 index 0000000..cef8ffd --- /dev/null +++ b/src/main/JUTests/AllClientServerTests.java @@ -0,0 +1,18 @@ +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.junit.runners.Suite.SuiteClasses; + +import data.io.ldap.LDAPDataReaderTest; +import data.io.ldap.LDAPDataWriterTest; +import data.io.sql.SQLRelDataReaderTest; + + +@RunWith(Suite.class) +@SuiteClasses({ + // SSSync_Connectors + LDAPDataReaderTest.class, LDAPDataWriterTest.class, + SQLRelDataReaderTest.class, +}) +public class AllClientServerTests { + +} diff --git a/src/main/JUTests/AllLocalTests.java b/src/main/JUTests/AllLocalTests.java new file mode 100644 index 0000000..bc9019d --- /dev/null +++ b/src/main/JUTests/AllLocalTests.java @@ -0,0 +1,29 @@ + + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.junit.runners.Suite.SuiteClasses; + +import sync.BasicSyncTaskTest; + +import conf.SSSyncConfParserTest; + +import data.MVDataEntryTest; +import data.io.SafeDataReaderTest; +import data.io.csv.CSVDataReaderTest; +import data.io.filters.MVDataCombinerTest; + +@RunWith(Suite.class) +@SuiteClasses( { + // SSSync + SSSyncConfParserTest.class, + SafeDataReaderTest.class, + BasicSyncTaskTest.class, + // SSSync_Connectors (only local) + CSVDataReaderTest.class, + // SSSync_Core + MVDataEntryTest.class, MVDataCombinerTest.class, + } ) +public class AllLocalTests { + +} diff --git a/src/main/JUTests/conf/SSSyncConfParserTest.java b/src/main/JUTests/conf/SSSyncConfParserTest.java new file mode 100644 index 0000000..100df16 --- /dev/null +++ b/src/main/JUTests/conf/SSSyncConfParserTest.java @@ -0,0 +1,69 @@ +package conf; + +import static org.junit.Assert.*; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.net.URL; + +import org.junit.Before; +import org.junit.Test; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; + +public class SSSyncConfParserTest { + + private File currentFolder; + + @Before + public void setup() { + URL main = SSSyncConfParserTest.class.getResource("SSSyncConfParserTest.class"); + if (!"file".equalsIgnoreCase(main.getProtocol())) + throw new IllegalStateException("This class is not stored in a file"); + currentFolder = new File(main.getPath()).getParentFile(); + } + + @Test + public void loadConfigTest() throws Exception { + + String expectedMain = readEntireFile(new File(currentFolder, "testExpectedMain.yaml")); + String expectedConn = readEntireFile(new File(currentFolder, "testExpectedConn.yaml")); + String mainConfigFile = new File(currentFolder, "testMain.yaml").getAbsolutePath(); + String connConfigFile = new File(currentFolder, "testConn.yaml").getAbsolutePath(); + + // Loading (config => beans) + ConfigRootBean confMain = SSSyncConfParser.loadMainConfig(mainConfigFile); + ConfigConnectionsBean confConn = SSSyncConfParser.loadConnConfig(connConfigFile); + + + System.out.println(confMain); + System.out.println(confConn); + + // Dumping (beans => config) + DumperOptions options = new DumperOptions(); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + Yaml yamlDump = new Yaml(options); + String dumpMain = yamlDump.dump(confMain); + String dumpConn = yamlDump.dump(confConn); + + // Checking that everything is kept + assertEquals(expectedMain, dumpMain); + assertEquals(expectedConn, dumpConn); + } + + private static String readEntireFile(File file) throws IOException { + FileReader in = new FileReader(file); + StringBuilder contents = new StringBuilder((int) file.length()); + char[] buffer = new char[4096]; + int read = 0; + do { + contents.append(buffer, 0, read); + read = in.read(buffer); + } while (read >= 0); + in.close(); + + return contents.toString(); + } + +} diff --git a/src/main/JUTests/conf/testConn.yaml b/src/main/JUTests/conf/testConn.yaml new file mode 100644 index 0000000..c41063c --- /dev/null +++ b/src/main/JUTests/conf/testConn.yaml @@ -0,0 +1,19 @@ +# This file contains credentials (should be readable only by SSSync) +connections: + - id : ora_1 + type: jdbc + dbms: oracle + ress: gest + host: ora.univ-jfc.fr + port: 1521 + user: GRHUM + pass: secret + db : GHRUM + + - id : ldap_1 + type: ldap + host: localhost + port: 389 + bind: uid=ldapadmin,ou=specialUsers,dc=univ-jfc,dc=fr + pass: secret + diff --git a/src/main/JUTests/conf/testExpectedConn.yaml b/src/main/JUTests/conf/testExpectedConn.yaml new file mode 100644 index 0000000..4cb3421 --- /dev/null +++ b/src/main/JUTests/conf/testExpectedConn.yaml @@ -0,0 +1,22 @@ +!!conf.ConfigConnectionsBean +connections: +- bind: null + db: GHRUM + dbms: oracle + host: ora.univ-jfc.fr + id: ora_1 + pass: secret + port: 1521 + ress: gest + type: jdbc + user: GRHUM +- bind: uid=ldapadmin,ou=specialUsers,dc=univ-jfc,dc=fr + db: null + dbms: null + host: localhost + id: ldap_1 + pass: secret + port: 389 + ress: null + type: ldap + user: null diff --git a/src/main/JUTests/conf/testExpectedMain.yaml b/src/main/JUTests/conf/testExpectedMain.yaml new file mode 100644 index 0000000..dd00aef --- /dev/null +++ b/src/main/JUTests/conf/testExpectedMain.yaml @@ -0,0 +1,70 @@ +!!conf.ConfigRootBean +globals: + maxExecTime: 3 +tasks: +- destination: + attr: uid + base: ou=people,dc=univ-jfc,dc=fr + conn: ldap_1 + kind: ldap + mode: null + name: LDAP de test, ou=people + path: null + query: null + name: People sync + opLimits: + delete: 10 + insert: 100 + update: 10 + skipEntryDelete: false + skipReadErrors: false + sources: + - attr: null + base: null + conn: ora_1 + kind: sql + mode: PRIMARY_SOURCE + name: GHRUM, comptes et personnes + path: null + query: people.sql + - attr: null + base: null + conn: null + kind: csv + mode: MERGE_APPEND + name: CSV personnes additionnelles + path: people_append.csv + query: null + - attr: null + base: null + conn: null + kind: sorted_csv + mode: MERGE_REPLACE + name: CSV correctifs personnes + path: people_replace.csv + query: null +- destination: + attr: supannEntiteAffectation + base: ou=structures,dc=univ-jfc,dc=fr + conn: ldap_1 + kind: ldap + mode: null + name: LDAP de test, ou=structures + path: null + query: null + name: Structure sync + opLimits: + delete: 10 + insert: 10 + update: 10 + skipEntryDelete: true + skipReadErrors: true + sources: + - attr: null + base: null + conn: ora_1 + kind: sql + mode: PRIMARY_SOURCE + name: GHRUM, structures + path: null + query: structures.sql diff --git a/src/main/JUTests/conf/testMain.yaml b/src/main/JUTests/conf/testMain.yaml new file mode 100644 index 0000000..39350b2 --- /dev/null +++ b/src/main/JUTests/conf/testMain.yaml @@ -0,0 +1,54 @@ +# This YAML file describe all synchronization tasks, with their readers and writers + +globals: + maxExecTime: 3 + +tasks: + - name: People sync + opLimits: + insert: 100 + update: 10 + delete: 10 + sources: + - name: GHRUM, comptes et personnes + kind: sql + conn: ora_1 + mode: PRIMARY_SOURCE + query: people.sql + + - name: CSV personnes additionnelles + kind: csv + mode: MERGE_APPEND + path: people_append.csv + + - name: CSV correctifs personnes + kind: sorted_csv + mode: MERGE_REPLACE + path: people_replace.csv + + destination: + name: LDAP de test, ou=people + kind: ldap + conn: ldap_1 + attr: uid + base: ou=people,dc=univ-jfc,dc=fr + + - name: Structure sync + sources: + - name: GHRUM, structures + kind: sql + conn: ora_1 + mode: PRIMARY_SOURCE + query: structures.sql + destination: + name: LDAP de test, ou=structures + kind: ldap + conn: ldap_1 + attr: supannEntiteAffectation + base: ou=structures,dc=univ-jfc,dc=fr + skipEntryDelete: true + skipReadErrors: true + opLimits: + insert: 10 + update: 10 + delete: 10
\ No newline at end of file diff --git a/src/main/JUTests/data/io/SafeDataReaderTest.java b/src/main/JUTests/data/io/SafeDataReaderTest.java new file mode 100644 index 0000000..427004b --- /dev/null +++ b/src/main/JUTests/data/io/SafeDataReaderTest.java @@ -0,0 +1,51 @@ +package data.io; + +import static org.junit.Assert.*; + +import java.util.Iterator; + +import org.apache.log4j.PropertyConfigurator; +import org.junit.BeforeClass; +import org.junit.Test; + +import data.MVDataEntry; +import data.io.stub.StubDataReader; + +public class SafeDataReaderTest { + + private static final String LOG_PROPERTIES_FILE = "conf/log4j.properties"; + + @BeforeClass + public static void setup() { + PropertyConfigurator.configure(LOG_PROPERTIES_FILE); + } + + @Test + public void testNoErrors() { + MVDataEntry testEntries[] = new MVDataEntry[5]; + for (int i=0;i<5;i++) { + testEntries[i] = new MVDataEntry("line"+(i+1)); + testEntries[i].put("attr1", "value"+(i+1)); + } + + StubDataReader src = new StubDataReader("testNoSkipErrors_src", testEntries); + StubDataReader expected = new StubDataReader("testNoSkipErrors_expected", testEntries); + + SafeDataReader reader = new SafeDataReader(src, false); + + // Test twice to check if asking a new iterator "rewinds" correctly + for (int i=0;i<2;i++) { + //System.out.println("Loop " + (i+1)); + Iterator<MVDataEntry> readerIt = reader.iterator(); + for ( MVDataEntry e: expected) { + assertTrue(readerIt.hasNext()); + MVDataEntry r = readerIt.next(); + //System.out.println(e + " / " + r); + assertEquals(e, r); + } + assertFalse(readerIt.hasNext()); + } + } + + //TODO Real tests with messy input readers (null values, exception, hasNext/next() incoherence) +} diff --git a/src/main/JUTests/sync/BasicSyncTaskTest.java b/src/main/JUTests/sync/BasicSyncTaskTest.java new file mode 100644 index 0000000..88d9c98 --- /dev/null +++ b/src/main/JUTests/sync/BasicSyncTaskTest.java @@ -0,0 +1,129 @@ +package sync; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.io.StringReader; + +import org.apache.log4j.PropertyConfigurator; +import org.junit.BeforeClass; +import org.junit.Test; + +import data.MVDataEntry; +import data.filters.MVDataCombiner; +import data.filters.MVDataCombiner.MVDataCombineMode; +import data.io.MVDataReader; +import data.io.SafeDataReader; +import data.io.csv.CSVDataReader; +import data.io.stub.StubDataReader; +import data.io.stub.StubDataWriter; + +public class BasicSyncTaskTest { + + private static final String LOG_PROPERTIES_FILE = "conf/log4j.properties"; + + @BeforeClass + public static void setup() { + PropertyConfigurator.configure(LOG_PROPERTIES_FILE); + } + + @Test + public void test() throws IOException { + + // Input flows setup + MVDataEntry[] fakeEntries1 = new MVDataEntry[5]; + fakeEntries1[0] = new MVDataEntry("line1"); + fakeEntries1[0].put("hello", "world"); + + fakeEntries1[1] = new MVDataEntry("line2"); + fakeEntries1[1].put("bla", "hidden"); + fakeEntries1[1].put("hello", "merged"); + + fakeEntries1[2] = new MVDataEntry("line3"); + fakeEntries1[2].put("hello", "world"); + + fakeEntries1[3] = new MVDataEntry("line4"); + fakeEntries1[3].put("hello", "world"); + + fakeEntries1[4] = new MVDataEntry("line5"); + fakeEntries1[4].put("hello", "world"); + + + MVDataEntry[] fakeEntries2 = new MVDataEntry[3]; + fakeEntries2[0] = new MVDataEntry("line1"); + fakeEntries2[0].put("hello", "world"); + + fakeEntries2[1] = new MVDataEntry("line2"); + fakeEntries2[1].put("bla", "replaced"); + + fakeEntries2[2] = new MVDataEntry("line3"); + fakeEntries2[2].put("hello", "world"); + + + MVDataEntry[] fakeEntries3 = new MVDataEntry[5]; + fakeEntries3[0] = new MVDataEntry("line2"); + fakeEntries3[0].put("hello", "world"); + fakeEntries3[0].put("extra", "to be preserved"); + + fakeEntries3[1] = new MVDataEntry("line2b"); + fakeEntries3[1].put("to be", "removed", null); + + fakeEntries3[2] = new MVDataEntry("line4"); + fakeEntries3[2].put("hello", "world"); + fakeEntries3[2].put("extra", "to be preserved"); + + fakeEntries3[3] = new MVDataEntry("line5"); + fakeEntries3[3].splitAndPut("hello", "too;much;world", ";"); + + fakeEntries3[4] = new MVDataEntry("line6"); + fakeEntries3[4].put("to be", "removed"); + + StubDataReader fakeReader1 = new StubDataReader("testSrc1", fakeEntries1); + StubDataReader fakeReader2 = new StubDataReader("testSrc3", fakeEntries2); + StubDataReader fakeReader3 = new StubDataReader("testDst", fakeEntries3); + + MVDataReader readers[] = new MVDataReader[]{ + new SafeDataReader(fakeReader1,false), + new SafeDataReader( + new CSVDataReader("testSrc2", + new StringReader(CSVDataReader.CSV_DEMO), + false + ), false + ), + new SafeDataReader(fakeReader2,false), + }; + + MVDataCombineMode mergeModes[] = new MVDataCombineMode[]{ + MVDataCombineMode.PRIMARY_SOURCE, + MVDataCombineMode.MERGE_APPEND, + MVDataCombineMode.MERGE_REPLACE, + }; + + MVDataReader srcReader = new MVDataCombiner("testSrcComb", readers, mergeModes); + MVDataReader dstReader = fakeReader3; + + // Output flow setup + StubDataWriter dstWriter = new StubDataWriter(10); + + // Data sync'er initialization + BasicSyncTask task = new BasicSyncTask("task1", false, srcReader, dstReader, dstWriter); + task.setOperationLimits(10,10,10); + + // Data sync'er run + assertTrue(task.call()); + + // Expected outputs + String expectedDstOps = + "INSERT: {key=line1, attrValPairs={hello=[world], attr2=[csv1], from=[csv1, csv1bis]}}\n" + + "UPDATE: {key=line2, attrValPairs={hello=[the, merged, world, all], bla=[replaced]}}\n" + + "DELETE: {key=line2b, attrValPairs={to be=[removed]}}\n" + + "INSERT: {key=line3, attrValPairs={hello=[world]}}\n" + + // Line 4 must not be updated ! + "UPDATE: {key=line5, attrValPairs={hello=[world]}}\n" + + "DELETE: {key=line6, attrValPairs={to be=[removed]}}\n"; + + // Check results + assertEquals(expectedDstOps, dstWriter.toString()); + } + +} diff --git a/src/main/build.xml b/src/main/build.xml new file mode 100644 index 0000000..8847365 --- /dev/null +++ b/src/main/build.xml @@ -0,0 +1,101 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- WARNING: Eclipse auto-generated file. + Any modifications will be overwritten. + To include a user specific buildfile here, simply create one in the same + directory with the processing instruction <?eclipse.ant.import?> + as the first entry and export the buildfile again. --> +<project basedir="." default="build" name="SSSync_Main"> + <property environment="env"/> + <property name="ECLIPSE_HOME" value="../../../../../../usr/lib/eclipse"/> + <property name="SSSync_Core.location" value="../core"/> + <property name="SSSync_Connectors.location" value="../connectors"/> + <property name="debuglevel" value="source,lines,vars"/> + <property name="target" value="1.6"/> + <property name="source" value="1.6"/> + <path id="JUnit 4.libraryclasspath"> + <pathelement location="../../../../../../usr/share/eclipse/dropins/jdt/plugins/org.junit_4.8.2.dist/junit.jar"/> + <pathelement location="../../../../../../usr/share/eclipse/dropins/jdt/plugins/org.hamcrest.core_1.1.0.jar"/> + </path> + <path id="SSSync_Core.classpath"> + <pathelement location="${SSSync_Core.location}/bin"/> + <pathelement location="${SSSync_Core.location}/lib/guava-16.0.1.jar"/> + <path refid="JUnit 4.libraryclasspath"/> + </path> + <path id="SSSync_Connectors.classpath"> + <pathelement location="${SSSync_Connectors.location}/bin"/> + <path refid="SSSync_Core.classpath"/> + <path refid="JUnit 4.libraryclasspath"/> + <pathelement location="${SSSync_Connectors.location}/lib/commons-csv-1.0-SNAPSHOT.jar"/> + <pathelement location="${SSSync_Connectors.location}/lib/ojdbc6.jar"/> + <pathelement location="${SSSync_Connectors.location}/lib/mysql-connector-java-5.1.31-bin.jar"/> + <pathelement location="${SSSync_Connectors.location}/lib/unboundid-ldapsdk-se.jar"/> + </path> + <path id="SSSync_Main.classpath"> + <pathelement location="bin"/> + <path refid="JUnit 4.libraryclasspath"/> + <path refid="SSSync_Core.classpath"/> + <pathelement location="lib/log4j-1.2.17.jar"/> + <pathelement location="lib/snakeyaml-1.11.jar"/> + <path refid="SSSync_Connectors.classpath"/> + </path> + <target name="init"> + <mkdir dir="bin"/> + <copy includeemptydirs="false" todir="bin"> + <fileset dir="src"> + <exclude name="**/*.java"/> + </fileset> + </copy> + <copy includeemptydirs="false" todir="bin"> + <fileset dir="JUTests"> + <exclude name="**/*.java"/> + </fileset> + </copy> + </target> + <target name="clean"> + <delete dir="bin"/> + </target> + <target depends="clean" name="cleanall"> + <ant antfile="build.xml" dir="${SSSync_Core.location}" inheritAll="false" target="clean"/> + <ant antfile="build.xml" dir="${SSSync_Connectors.location}" inheritAll="false" target="clean"/> + </target> + <target depends="build-subprojects,build-project" name="build"/> + <target name="build-subprojects"> + <ant antfile="build.xml" dir="${SSSync_Core.location}" inheritAll="false" target="build-project"> + <propertyset> + <propertyref name="build.compiler"/> + </propertyset> + </ant> + <ant antfile="build.xml" dir="${SSSync_Connectors.location}" inheritAll="false" target="build-project"> + <propertyset> + <propertyref name="build.compiler"/> + </propertyset> + </ant> + </target> + <target depends="init" name="build-project"> + <echo message="${ant.project.name}: ${ant.file}"/> + <javac debug="true" debuglevel="${debuglevel}" destdir="bin" includeantruntime="false" source="${source}" target="${target}"> + <src path="src"/> + <src path="JUTests"/> + <classpath refid="SSSync_Main.classpath"/> + </javac> + </target> + <target description="Build all projects which reference this project. Useful to propagate changes." name="build-refprojects"/> + <target description="copy Eclipse compiler jars to ant lib directory" name="init-eclipse-compiler"> + <copy todir="${ant.library.dir}"> + <fileset dir="${ECLIPSE_HOME}/plugins" includes="org.eclipse.jdt.core_*.jar"/> + </copy> + <unzip dest="${ant.library.dir}"> + <patternset includes="jdtCompilerAdapter.jar"/> + <fileset dir="${ECLIPSE_HOME}/plugins" includes="org.eclipse.jdt.core_*.jar"/> + </unzip> + </target> + <target description="compile project with Eclipse compiler" name="build-eclipse-compiler"> + <property name="build.compiler" value="org.eclipse.jdt.core.JDTCompilerAdapter"/> + <antcall target="build"/> + </target> + <target name="SSSync"> + <java classname="SSSync" failonerror="true" fork="yes"> + <classpath refid="SSSync_Main.classpath"/> + </java> + </target> +</project> diff --git a/src/main/conf/connections.yaml b/src/main/conf/connections.yaml new file mode 100644 index 0000000..1918d02 --- /dev/null +++ b/src/main/conf/connections.yaml @@ -0,0 +1,18 @@ +# This file contains credentials (should be readable only by SSSync) +connections: + - id : mysql_1 + type: jdbc + dbms: mysql + host: localhost + port: 3306 + user: root + pass: secret + db : sssync + + - id : ldap_1 + type: ldap + host: localhost + port: 389 + bind: uid=ldapadmin,ou=specialUsers,dc=univ-jfc,dc=fr + pass: secret + diff --git a/src/main/conf/log4j.properties b/src/main/conf/log4j.properties new file mode 100644 index 0000000..6dccde8 --- /dev/null +++ b/src/main/conf/log4j.properties @@ -0,0 +1,29 @@ +# +# our log4j properties / configuration file +# +# STDOUT appender +log4j.appender.STDOUT=org.apache.log4j.ConsoleAppender +log4j.appender.STDOUT.layout=org.apache.log4j.PatternLayout +log4j.appender.STDOUT.layout.ConversionPattern=%d %p [%t] %C{1} - %m\n + + +# Normal operation mode +log4j.category.SSSync=INFO, STDOUT +# Configuration dump +#log4j.category.SSSync=DEBUG, STDOUT + +# Normal operation mode (currently nothing more in DEBUG or TRACE modes) +log4j.category.data.io.SafeDataReader=INFO, STDOUT + +# Normal operation mode +#log4j.category.sync.BasicSyncTask=INFO, STDOUT +# Trace insert/update/delete operation +log4j.category.sync.BasicSyncTask=DEBUG, STDOUT +# Trace every key comparison +#log4j.category.sync.BasicSyncTask=TRACE, STDOUT + +# Keep silent about memory and GC +log4j.category.utils.JVMStatsDumper=INFO, STDOUT +# Trace memory usage/GC + dump configuration +#log4j.category.utils.JVMStatsDumper=DEBUG, STDOUT + diff --git a/src/main/conf/queries/people.sql b/src/main/conf/queries/people.sql new file mode 100644 index 0000000..ab66d5f --- /dev/null +++ b/src/main/conf/queries/people.sql @@ -0,0 +1,5 @@ +SELECT + p.*, + "person;posixAccount;top" as objectClass +FROM sssync.people p +ORDER BY 1 ASC; diff --git a/src/main/conf/queries/structures.sql b/src/main/conf/queries/structures.sql new file mode 100644 index 0000000..626273c --- /dev/null +++ b/src/main/conf/queries/structures.sql @@ -0,0 +1,5 @@ +SELECT + s.*, + "supannEntite;organizationalUnit;top" as objectClass +FROM sssync.structures s +ORDER BY 1 ASC; diff --git a/src/main/conf/sssync.yaml b/src/main/conf/sssync.yaml new file mode 100644 index 0000000..b285a37 --- /dev/null +++ b/src/main/conf/sssync.yaml @@ -0,0 +1,56 @@ +# This YAML file describe all synchronization tasks, with their readers and writers + +globals: + maxExecTime: 3 # minutes + +tasks: + - name: People sync + opLimits: + insert: 300 + update: 300 + delete: 300 + sources: + - name: GHRUM, comptes et personnes + kind: sql + conn: mysql_1 + mode: PRIMARY_SOURCE + query: conf/queries/people.sql + + - name: CSV personnes additionnelles + kind: csv + mode: MERGE_APPEND + path: data/people_append.csv + + - name: CSV correctifs personnes + kind: csv + mode: MERGE_REPLACE + path: data/people_replace.csv + + destination: + name: LDAP de test, ou=people + kind: ldap + conn: ldap_1 + attr: uid + base: ou=people,dc=univ-jfc,dc=fr + + - name: Structure sync + opLimits: + insert: 10 + update: 10 + delete: 10 + sources: + - name: GHRUM, structures + kind: sql + conn: mysql_1 + mode: PRIMARY_SOURCE + query: conf/queries/structures.sql + + destination: + name: LDAP de test, ou=structures + kind: ldap + conn: ldap_1 + attr: supannCodeEntite + base: ou=structures,dc=univ-jfc,dc=fr + + skipEntryDelete: true + skipReadErrors: true
\ No newline at end of file diff --git a/src/main/data/people_append.csv b/src/main/data/people_append.csv new file mode 100644 index 0000000..dc526ff --- /dev/null +++ b/src/main/data/people_append.csv @@ -0,0 +1 @@ +lpouzenc,cn,Second-prénom
\ No newline at end of file diff --git a/src/main/data/people_replace.csv b/src/main/data/people_replace.csv new file mode 100644 index 0000000..372ed67 --- /dev/null +++ b/src/main/data/people_replace.csv @@ -0,0 +1,3 @@ +lpouzenc,loginShell,/bin/ksh +,, + diff --git a/src/main/lib/log4j-1.2.17.jar b/src/main/lib/log4j-1.2.17.jar Binary files differnew file mode 100644 index 0000000..068867e --- /dev/null +++ b/src/main/lib/log4j-1.2.17.jar diff --git a/src/main/lib/snakeyaml-1.11-javadoc.jar b/src/main/lib/snakeyaml-1.11-javadoc.jar Binary files differnew file mode 100644 index 0000000..bac2a05 --- /dev/null +++ b/src/main/lib/snakeyaml-1.11-javadoc.jar diff --git a/src/main/lib/snakeyaml-1.11.jar b/src/main/lib/snakeyaml-1.11.jar Binary files differnew file mode 100644 index 0000000..3e237cd --- /dev/null +++ b/src/main/lib/snakeyaml-1.11.jar diff --git a/src/main/src/SSSync.java b/src/main/src/SSSync.java new file mode 100644 index 0000000..422c31e --- /dev/null +++ b/src/main/src/SSSync.java @@ -0,0 +1,208 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.PropertyConfigurator; + +import conf.ConfigConnectionsBean; +import conf.ConfigGlobalsBean; +import conf.ConfigRootBean; +import conf.SSSyncConfParser; +import conf.SSSyncConnectionsFactory; +import conf.SSSyncTasksFactory; +import data.io.ConnectionsHolder; + +import sync.BasicSyncTask; +import utils.JVMStatsDumper; + +/** + * Main class for Simple and Stupid Sync'er + * + * @author lpouzenc + */ +public class SSSync { + private static final Logger logger = Logger.getLogger(SSSync.class.getName()); + + private static final String LOG_PROPERTIES_FILE = "conf/log4j.properties"; + private static final String CONFIG_MAIN_FILE = "conf/sssync.yaml"; + private static final String CONFIG_CONN_FILE = "conf/connections.yaml"; + + private static final int ERR_SUCCESS = 0; + private static final int ERR_CONFIG_PARSE_ERROR = 1; + private static final int ERR_CONN_INIT_ERROR = 2; + private static final int ERR_TASK_INIT_ERROR = 3; + private static final int ERR_DRYRUN_FAILURE = 4; + private static final int ERR_REALRUN_FAILURE = 5; + //TODO private static final int ERR_MAXTIME_REACHED = 6; + + /** + * Main entry point. Takes care of cmdline parsing, config files interpretation, + * tasks setup and start. + * + * @param args + */ + public static void main(String[] args) { + // log4j setup (first thing to do) + PropertyConfigurator.configure(LOG_PROPERTIES_FILE); + logger.info("Program start (user: '" + System.getProperty("user.name") + + "', cwd: '" + System.getProperty("user.dir") + "')"); + + //TODO use cmdline args for config file path + String mainConfigFile = CONFIG_MAIN_FILE; + String connConfigFile = CONFIG_CONN_FILE; + + // Config parsing + ConfigRootBean confMain = null; + ConfigConnectionsBean confConn = null; + try { + confMain = SSSyncConfParser.loadMainConfig(mainConfigFile); + confConn = SSSyncConfParser.loadConnConfig(connConfigFile); + } catch (Exception e) { + logger.fatal("Exception while loading configuration", e); + end(ERR_CONFIG_PARSE_ERROR); + } + ConfigGlobalsBean confGlobals = confMain.getGlobals(); + + // Config dump if DEBUG level (or finer) + if ( !logger.getLevel().isGreaterOrEqual(Level.INFO) ) { + logger.debug("Current connection configuration :\n" + confConn); + logger.debug("Current main configuration :\n" + confMain); + } + + // Connections init + logger.info("Connections initialization"); + ConnectionsHolder connections = null; + try { + connections = SSSyncConnectionsFactory.setupConnections(confConn); + } catch (Exception e) { + logger.fatal("Exception while establishing connections", e); + end(ERR_CONN_INIT_ERROR); + } + + // Suggest garbage collector to forget our passwords since we are connected + confConn=null; + System.gc(); + JVMStatsDumper.logMemoryUsage(); + + + // Tasks init + logger.info("Tasks initialization"); + List<BasicSyncTask> tasks = null; + try { + tasks = SSSyncTasksFactory.setupTasks(connections, confMain); + } catch (Exception e) { + logger.fatal("Exception during tasks initialization", e); + end(ERR_TASK_INIT_ERROR); + } + + logger.info("Tasks are ready to start"); + JVMStatsDumper.logMemoryUsage(); + + + // Tasks first (dry) run + if ( ! SSSync.safeTaskRun(tasks, confGlobals.getMaxExecTime(), true) ) { + logger.error("Dry-run pass has shown problems, skipping real synchronization"); + end(ERR_DRYRUN_FAILURE); + } + + // Tasks second (real) run + if ( SSSync.safeTaskRun(tasks, confGlobals.getMaxExecTime(), false) ) { + logger.error("Real-run pass has shown problems, data could be messed up !"); + end(ERR_REALRUN_FAILURE); + } + + // Clean-up + try { + connections.close(); + } catch (IOException e) { + logger.info("Problem during connections closing"); + } + + // Normal exit + end(ERR_SUCCESS); + } + + /** + * Method to run safely a sequence of tasks within a given time period. + * In a separate thread, it runs all the tasks sequentially. + * + * @param list + * @param timeOutInMinute + * @return + * @throws ExecutionException + * @throws InterruptedException + */ + private static boolean safeTaskRun(List<BasicSyncTask> list, long timeOutInMinute, boolean dryRun) { + ExecutorService executor = Executors.newSingleThreadExecutor(); + List<Future<Boolean>> results; + boolean aborted = false; + + logger.info("Starting " + (dryRun?"dry-run":"real-run") + " synchronization pass"); + + for ( BasicSyncTask t : list ) { + t.setDryRun(dryRun); + } + + try { + results = executor.invokeAll(list, timeOutInMinute, TimeUnit.MINUTES); + // Join all tasks, seeking for an unsuccessful execution + for (Future<Boolean> r: results) { + if ( ! r.get() ) { + aborted = true; + } + } + } catch (CancellationException e) { + logger.fatal("Global maximum execution time exhausted, aborting tasks !"); + aborted = true; + } catch (InterruptedException e) { + logger.fatal("Worker thread for task execution was interrupted", e); + aborted = true; + } catch (ExecutionException e) { + logger.error("Exception during tasks execution", e.getCause()); + aborted = true; + } + + JVMStatsDumper.logMemoryUsage(); + executor.shutdown(); + + return !aborted; + } + + /** + * Helper function to always log the end of program + * @param result + */ + private static void end(int result) { + JVMStatsDumper.logGCStats(); + logger.info("Program end (result code: " + result + ")"); + System.exit(result); + } + +} diff --git a/src/main/src/conf/ConfigConnectionBean.java b/src/main/src/conf/ConfigConnectionBean.java new file mode 100644 index 0000000..b43b56f --- /dev/null +++ b/src/main/src/conf/ConfigConnectionBean.java @@ -0,0 +1,111 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package conf; + +import data.io.sql.SQLConnectionWrapper.DBMSType; + +/** + * Generated Configuration Bean + */ +public class ConfigConnectionBean { + + public enum ConnectionType { jdbc, ldap } + + private String id; + private ConnectionType type; + private DBMSType dbms; + private String ress; + private String host; + private int port; + private String user; + private String bind; + private String pass; + private String db; + + public String getId() { + return id; + } + public void setId(String id) { + this.id = id; + } + public ConnectionType getType() { + return type; + } + public void setType(ConnectionType type) { + this.type = type; + } + public DBMSType getDbms() { + return dbms; + } + public void setDbms(DBMSType dbms) { + this.dbms = dbms; + } + public String getRess() { + return ress; + } + public void setRess(String ress) { + this.ress = ress; + } + public String getHost() { + return host; + } + public void setHost(String host) { + this.host = host; + } + public int getPort() { + return port; + } + public void setPort(int port) { + this.port = port; + } + public String getUser() { + return user; + } + public void setUser(String user) { + this.user = user; + } + public String getBind() { + return bind; + } + public void setBind(String bind) { + this.bind = bind; + } + public String getPass() { + return pass; + } + public void setPass(String pass) { + this.pass = pass; + } + public String getDb() { + return db; + } + public void setDb(String db) { + this.db = db; + } + + @Override + public String toString() { + return "ConfigConnectionBean [id=" + id + ", type=" + type + ", dbms=" + dbms + + ", ress=" + ress + ", host=" + host + ", port=" + port + + ", user=" + user + ", bind=" + bind + ", pass=(obfuscated)]"; + } + +} diff --git a/src/main/src/conf/ConfigConnectionsBean.java b/src/main/src/conf/ConfigConnectionsBean.java new file mode 100644 index 0000000..9fb034b --- /dev/null +++ b/src/main/src/conf/ConfigConnectionsBean.java @@ -0,0 +1,45 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package conf; + +import java.util.List; + +/** + * Generated Configuration Bean + */ +public class ConfigConnectionsBean { + + private List<ConfigConnectionBean> connections; + + public List<ConfigConnectionBean> getConnections() { + return connections; + } + + public void setConnections(List<ConfigConnectionBean> connections) { + this.connections = connections; + } + + @Override + public String toString() { + return "ConfigConnectionsBean [connections=" + ConfigRootBean.listDump(connections,1) + "]"; + } + +} diff --git a/src/main/src/conf/ConfigGlobalsBean.java b/src/main/src/conf/ConfigGlobalsBean.java new file mode 100644 index 0000000..256acee --- /dev/null +++ b/src/main/src/conf/ConfigGlobalsBean.java @@ -0,0 +1,41 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package conf; + +/** + * Generated Configuration Bean + */ +public class ConfigGlobalsBean { + private int maxExecTime; + + public int getMaxExecTime() { + return maxExecTime; + } + + public void setMaxExecTime(int maxExecTime) { + this.maxExecTime = maxExecTime; + } + + @Override + public String toString() { + return "ConfigGlobalsBean [maxExecTime=" + maxExecTime + "]"; + } +} diff --git a/src/main/src/conf/ConfigOpLimitsBean.java b/src/main/src/conf/ConfigOpLimitsBean.java new file mode 100644 index 0000000..8f68e8c --- /dev/null +++ b/src/main/src/conf/ConfigOpLimitsBean.java @@ -0,0 +1,55 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package conf; + +/** + * Generated Configuration Bean + */ +public class ConfigOpLimitsBean { + private int insert; + private int update; + private int delete; + + public int getInsert() { + return insert; + } + public void setInsert(int insert) { + this.insert = insert; + } + public int getUpdate() { + return update; + } + public void setUpdate(int update) { + this.update = update; + } + public int getDelete() { + return delete; + } + public void setDelete(int delete) { + this.delete = delete; + } + + @Override + public String toString() { + return "ConfigOpLimitsBean [insert=" + insert + ", update=" + update + + ", delete=" + delete + "]"; + } +} diff --git a/src/main/src/conf/ConfigRootBean.java b/src/main/src/conf/ConfigRootBean.java new file mode 100644 index 0000000..acbbd49 --- /dev/null +++ b/src/main/src/conf/ConfigRootBean.java @@ -0,0 +1,73 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package conf; + +import java.util.List; + +/** + * Generated Configuration Bean + */ +public class ConfigRootBean { + + private ConfigGlobalsBean globals; + private List<ConfigTaskBean> tasks; + + public ConfigGlobalsBean getGlobals() { + return globals; + } + public void setGlobals(ConfigGlobalsBean globals) { + this.globals = globals; + } + + public List<ConfigTaskBean> getTasks() { + return tasks; + } + public void setTasks(List<ConfigTaskBean> tasks) { + this.tasks = tasks; + } + + @Override + public String toString() { + return "ConfigRootBean [globals=" + globals + ", tasks=" + listDump(tasks, 1) + "]"; + } + + + public static <T> String listDump(List<T> list, int ident) { + StringBuffer buf = new StringBuffer(); + buf.append('{'); + for (T item : list) { + buf.append('\n'); + for (int i = 0; i < ident; i++) { + buf.append('\t'); + } + buf.append(item.toString()); + buf.append(','); + } + buf.append('\n'); + for (int i = 0; i < ident-1; i++) { + buf.append('\t'); + } + buf.append('}'); + return buf.toString(); + } + + +} diff --git a/src/main/src/conf/ConfigSrcOrDestBean.java b/src/main/src/conf/ConfigSrcOrDestBean.java new file mode 100644 index 0000000..5be1674 --- /dev/null +++ b/src/main/src/conf/ConfigSrcOrDestBean.java @@ -0,0 +1,96 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package conf; + +import data.filters.MVDataCombiner; + +/** + * Generated Configuration Bean + */ +public class ConfigSrcOrDestBean { + + public enum SourceKind { csv, ldap, sorted_csv, sql }; + + private String name; + private SourceKind kind; + private String conn; + private MVDataCombiner.MVDataCombineMode mode; + private String query; + private String path; + private String attr; + private String base; + + public String getName() { + return name; + } + public void setName(String name) { + this.name = name; + } + public SourceKind getKind() { + return kind; + } + public void setKind(SourceKind kind) { + this.kind = kind; + } + public String getConn() { + return conn; + } + public void setConn(String conn) { + this.conn = conn; + } + public MVDataCombiner.MVDataCombineMode getMode() { + return mode; + } + public void setMode(MVDataCombiner.MVDataCombineMode mode) { + this.mode = mode; + } + public String getQuery() { + return query; + } + public void setQuery(String query) { + this.query = query; + } + public String getPath() { + return path; + } + public void setPath(String path) { + this.path = path; + } + public String getAttr() { + return attr; + } + public void setAttr(String attr) { + this.attr = attr; + } + public String getBase() { + return base; + } + public void setBase(String base) { + this.base = base; + } + + @Override + public String toString() { + return "ConfigSrcOrDestBean [name=" + name + ", kind=" + kind + + ", conn=" + conn + ", mode=" + mode + ", query=" + query + + ", path=" + path + ", attr=" + attr + ", base=" + base + "]"; + } +} diff --git a/src/main/src/conf/ConfigTaskBean.java b/src/main/src/conf/ConfigTaskBean.java new file mode 100644 index 0000000..ed34eee --- /dev/null +++ b/src/main/src/conf/ConfigTaskBean.java @@ -0,0 +1,80 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package conf; + +import java.util.List; + +/** + * Generated Configuration Bean + */ +public class ConfigTaskBean { + + private String name; + private ConfigOpLimitsBean opLimits; + private List<ConfigSrcOrDestBean> sources; + private ConfigSrcOrDestBean destination; + private boolean skipReadErrors; + private boolean skipEntryDelete; + + public String getName() { + return name; + } + public void setName(String name) { + this.name = name; + } + public ConfigOpLimitsBean getOpLimits() { + return opLimits; + } + public void setOpLimits(ConfigOpLimitsBean opLimits) { + this.opLimits = opLimits; + } + public List<ConfigSrcOrDestBean> getSources() { + return sources; + } + public void setSources(List<ConfigSrcOrDestBean> sources) { + this.sources = sources; + } + public ConfigSrcOrDestBean getDestination() { + return destination; + } + public void setDestination(ConfigSrcOrDestBean destination) { + this.destination = destination; + } + public boolean isSkipReadErrors() { + return skipReadErrors; + } + public void setSkipReadErrors(boolean skipReadErrors) { + this.skipReadErrors = skipReadErrors; + } + public boolean isSkipEntryDelete() { + return skipEntryDelete; + } + public void setSkipEntryDelete(boolean skipDelete) { + this.skipEntryDelete = skipDelete; + } + @Override + public String toString() { + return "ConfigTaskBean [name=" + name + ", opLimits=" + opLimits + + ", sources=" + sources + ", destination=" + destination + + ", skipReadErrors=" + skipReadErrors + ", skipEntryDelete=" + + skipEntryDelete + "]"; + } +} diff --git a/src/main/src/conf/SSSyncConfParser.java b/src/main/src/conf/SSSyncConfParser.java new file mode 100644 index 0000000..42dc760 --- /dev/null +++ b/src/main/src/conf/SSSyncConfParser.java @@ -0,0 +1,65 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package conf; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.text.ParseException; + +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.Constructor; + +/** + * TODO javadoc + * + * @author lpouzenc + */ +public class SSSyncConfParser { + + public static ConfigRootBean loadMainConfig(String mainConfigFile) throws FileNotFoundException, ParseException { + Yaml yamlMain = new Yaml(new Constructor(ConfigRootBean.class)); + + //TODO : try to prevent weird exceptions when config is not respecting the implicit grammar of the bean tree + + ConfigRootBean confMain = (ConfigRootBean) yamlMain.load(new FileInputStream(mainConfigFile)); + + if ( confMain == null || confMain.getGlobals() == null ) { + throw new ParseException("Config parser has returned a null item", 0); + } + + // TODO : check config sanity and completeness + + return confMain; + } + + public static ConfigConnectionsBean loadConnConfig(String connConfigFile) throws FileNotFoundException, ParseException { + Yaml yamlConn = new Yaml(new Constructor(ConfigConnectionsBean.class)); + + ConfigConnectionsBean confConn = (ConfigConnectionsBean) yamlConn.load(new FileInputStream(connConfigFile)); + + if ( confConn == null ) { + throw new ParseException("Config parser has return a null item", 0); + } + + return confConn; + } + +} diff --git a/src/main/src/conf/SSSyncConnectionsFactory.java b/src/main/src/conf/SSSyncConnectionsFactory.java new file mode 100644 index 0000000..e747258 --- /dev/null +++ b/src/main/src/conf/SSSyncConnectionsFactory.java @@ -0,0 +1,61 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package conf; + +import data.io.ConnectionsHolder; +import data.io.ldap.LDAPConnectionWrapper; +import data.io.sql.SQLConnectionWrapper; + +/** + * TODO javadoc + * + * @author lpouzenc + */ +public class SSSyncConnectionsFactory { + + /** + * Setup all connections described in config + * @return + * @throws Exception + */ + public static ConnectionsHolder setupConnections(ConfigConnectionsBean confConn) throws Exception { + ConnectionsHolder connections = new ConnectionsHolder(); + + for ( ConfigConnectionBean conn : confConn.getConnections() ) { + switch (conn.getType()) { + case jdbc: + SQLConnectionWrapper connSQL = new SQLConnectionWrapper(conn.getDbms(), conn.getHost(), conn.getPort(), conn.getRess(), conn.getUser(), conn.getPass(), conn.getDb()); + connections.putConnSQL(conn.getId(), connSQL); + break; + case ldap: + LDAPConnectionWrapper connLDAP = new LDAPConnectionWrapper(conn.getHost(), conn.getPort(), conn.getBind(), conn.getPass()); + connections.putConnLDAP(conn.getId(), connLDAP); + break; + default: + //XXX : find better Exception type + throw new Exception("Bad config : conn '" + conn.getId() + "' unsupported type"); + } + } + + return connections; + } + +} diff --git a/src/main/src/conf/SSSyncTasksFactory.java b/src/main/src/conf/SSSyncTasksFactory.java new file mode 100644 index 0000000..de3e8f6 --- /dev/null +++ b/src/main/src/conf/SSSyncTasksFactory.java @@ -0,0 +1,147 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package conf; + +import java.io.File; +import java.io.FileReader; +import java.util.ArrayList; +import java.util.List; + +import sync.BasicSyncTask; +import data.filters.MVDataCombiner; +import data.filters.MVDataCombiner.MVDataCombineMode; +import data.io.ConnectionsHolder; +import data.io.MVDataReader; +import data.io.MVDataWriter; +import data.io.SafeDataReader; +import data.io.csv.CSVDataReader; +import data.io.ldap.LDAPConnectionWrapper; +import data.io.sql.SQLConnectionWrapper; + +/** + * TODO javadoc + * + * @author lpouzenc + */ +public class SSSyncTasksFactory { + + /** + * Build tasks objects with all needed resources from a config beans tree + * @param conf + * @return + * @throws Exception + */ + public static List<BasicSyncTask> setupTasks(ConnectionsHolder connections, ConfigRootBean confMain) throws Exception { + List<BasicSyncTask> tasks = new ArrayList<BasicSyncTask>(); + + // For each task... + for ( ConfigTaskBean confTask: confMain.getTasks() ) { + MVDataReader srcReader=null; + + // Building all sources + + List<ConfigSrcOrDestBean> confSources = confTask.getSources(); + // See if we are in multiple source situation (then MVDataCombiner) or not (then simple MVDataReader) + if ( confSources.size() == 0 ) { + throw new Exception("Bad config : task '" + confTask.getName() + "' has no defined sources"); + } else if ( confSources.size() == 1 ) { + srcReader = new SafeDataReader(_makeReader(connections, confSources.get(0), confTask.getName()), confTask.isSkipReadErrors()); + } else { + List<MVDataReader> readers = new ArrayList<MVDataReader>(); + List<MVDataCombineMode> mergeModes = new ArrayList<MVDataCombineMode>(); + + // For each source of the future MVDataCombiner... + for ( ConfigSrcOrDestBean confSource: confSources ) { + // Create and add the reader and his parameters + readers.add(new SafeDataReader(_makeReader(connections, confSource, confTask.getName()), confTask.isSkipReadErrors())); + mergeModes.add(confSource.getMode()); + } + + srcReader = new MVDataCombiner("srcCombiner", readers.toArray(new MVDataReader[0]), mergeModes.toArray(new MVDataCombineMode[0])); + } + + // Building destination + + MVDataReader dstReader=null; + MVDataWriter dstWriter=null; + + ConfigSrcOrDestBean confDestination = confTask.getDestination(); + switch ( confDestination.getKind() ) { + case ldap: + LDAPConnectionWrapper builder = connections.getLDAPConnectionBuilder(confDestination.getConn()); + // TODO : configurable lookAhead + MVDataReader tmpReader = builder.newFlatReader(confDestination.getName()+"_reader", confDestination.getBase(), confDestination.getAttr(), 128); + dstReader = new SafeDataReader(tmpReader, false); + dstWriter = builder.newFlatWriter(confDestination.getBase(), confDestination.getAttr()); + break; + default: + throw new Exception("Bad config : task '" + confTask.getName() + "' unsupported destination kind"); + } + + // Then building the sync task and add it to the task list + int maxInserts = confTask.getOpLimits().getInsert(); + int maxUpdates = confTask.getOpLimits().getUpdate(); + int maxDeletes = confTask.getOpLimits().getDelete(); + + BasicSyncTask task = new BasicSyncTask(confTask.getName(), false, srcReader, dstReader, dstWriter); + task.setOperationLimits(maxInserts, maxUpdates, maxDeletes); + task.setSkipEntryDelete(confTask.isSkipEntryDelete()); + tasks.add(task); + } + + return tasks; + } + + /** + * Helper function to make a new reader from an existing connection + * @param confSource + * @param taskName + * @return + * @throws Exception + */ + private static MVDataReader _makeReader(ConnectionsHolder connections, ConfigSrcOrDestBean confSource, String taskName) throws Exception { + MVDataReader reader=null; + switch (confSource.getKind()) { + case csv: + reader = new CSVDataReader(confSource.getName(), new FileReader(confSource.getPath()), false); + break; + case ldap: + LDAPConnectionWrapper ldapConnBuilder = connections.getLDAPConnectionBuilder(confSource.getConn()); + //FIXME : if conf error, get...ConnectionBuilder could return null + //TODO : configurable lookAhead + reader = ldapConnBuilder.newFlatReader(confSource.getName(), confSource.getBase(), confSource.getAttr(), 128); + break; + case sorted_csv: + reader = new CSVDataReader(confSource.getName(), new FileReader(confSource.getPath()), true); + break; + case sql: + SQLConnectionWrapper sqlConnBuilder = connections.getSQLConnectionBuilder(confSource.getConn()); + //TODO We assume the query config item is a filepath. It isn't checked anywhere. + reader = sqlConnBuilder.newReader(confSource.getName(), new File(confSource.getQuery())); + break; + default: + throw new Exception("Bad config : task '" + taskName + "' unsupported source kind"); + } + + return reader; + } + +} diff --git a/src/main/src/data/io/ConnectionsHolder.java b/src/main/src/data/io/ConnectionsHolder.java new file mode 100644 index 0000000..3a6e527 --- /dev/null +++ b/src/main/src/data/io/ConnectionsHolder.java @@ -0,0 +1,81 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.io; + +import java.io.Closeable; +import java.io.IOException; +import java.util.HashMap; + +import data.io.ldap.LDAPConnectionWrapper; +import data.io.sql.SQLConnectionWrapper; + +/** + * TODO javadoc + * + * @author lpouzenc + */ +public class ConnectionsHolder implements Closeable { + + public final HashMap<String, LDAPConnectionWrapper> connMapLDAP; + public final HashMap<String, SQLConnectionWrapper> connMapSQL; + + //TODO : with some refactoring, this class may disappear + /** + * Bean class to keep track of all opened connections in a single object + */ + public ConnectionsHolder() { + this.connMapLDAP = new HashMap<String, LDAPConnectionWrapper>(); + this.connMapSQL = new HashMap<String, SQLConnectionWrapper>(); + } + + public LDAPConnectionWrapper getLDAPConnectionBuilder(String conn) { + return connMapLDAP.get(conn); + } + + public SQLConnectionWrapper getSQLConnectionBuilder(String conn) { + return connMapSQL.get(conn); + } + + public void putConnLDAP(String connId, LDAPConnectionWrapper connLDAP) { + this.connMapLDAP.put(connId, connLDAP); + } + + public void putConnSQL(String connId, SQLConnectionWrapper connSQL) { + this.connMapSQL.put(connId, connSQL); + } + + /** + * Close all connections + */ + @Override + public void close() throws IOException { + // XXX : this will stop at first uncloseable connection. It isn't a very interesting problem however. + for ( LDAPConnectionWrapper connLDAP: connMapLDAP.values() ) { + connLDAP.close(); + } + for ( SQLConnectionWrapper connSQL: connMapSQL.values() ) { + connSQL.close(); + } + } + + + +} diff --git a/src/main/src/data/io/SafeDataReader.java b/src/main/src/data/io/SafeDataReader.java new file mode 100644 index 0000000..2c5dda9 --- /dev/null +++ b/src/main/src/data/io/SafeDataReader.java @@ -0,0 +1,155 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.io; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +import org.apache.log4j.Logger; + +import data.MVDataEntry; + +/** + * Multi-valued "safe" stream reader proxy. + * Adds logging and skipReadError mode feature. Check if items are well ordered. + * Ensures consistency of hasNext() / next() even if source stream is faulty. + * Never returns null items but throw NoSuchElementException if no other choices. + * + * @author lpouzenc + */ +public class SafeDataReader extends AbstractMVDataReader { + + private static final Logger logger = Logger.getLogger(SafeDataReader.class.getName()); + + private final MVDataReader src; + /** + * If true, continue even in case of read errors + */ + private final boolean skipReadErrors; + + private transient Iterator<MVDataEntry> srcIt; + private transient boolean abort; + private transient MVDataEntry previousData; + + + public SafeDataReader(MVDataReader src, boolean skipReadErrors) { + this.src = src; + this.dataSourceName = src.getDataSourceName(); + this.skipReadErrors = skipReadErrors; + } + + /** + * {@inheritDoc} + */ + @Override + public Iterator<MVDataEntry> iterator() { + // Reset everything + srcIt = src.iterator(); + abort = false; + previousData = null; + + return this; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean hasNext() { + return (!abort && srcIt.hasNext()); + } + + /** + * {@inheritDoc} + */ + @Override + public MVDataEntry next() { + boolean alreadyWarned=false; + boolean done=false; + MVDataEntry entry = null; + + // Prepare an hint for read exception (knowledge of last successfully read entry could help) + String hint = ( previousData != null )?previousData.getKey():"(nothing)"; + + // Seek for the next valid entry + while (!this.abort && !done && srcIt.hasNext()) { + + // Try to read next entry + try { + entry=src.next(); + if ( entry == null ) throw new NoSuchElementException("Null item returned"); + } catch (Exception e) { + logger.warn(src.getDataSourceName() + " : exception when seeking next valid entry after " + hint, e); + entry = null; // Make sure don't re-use a previous entry + } + + // Sanity checks + boolean valid = ( entry != null && entry.isValid() ); + //XXX Regex should be a parameter + if ( valid && !entry.getKey().matches("^\\p{Print}+$") ) { + logger.warn(src.getDataSourceName() + " : Invalid key found : '" + entry.getKey().replaceAll("[^\\p{Print}]", "?") + "' after " + hint); + valid = false; + } + + + // Two branches : If valid, check ordering then skip or done. If invalid : skip or abort. + if ( valid ) { + // Ensure that data.key is greater than previousData.key or abort + if ( previousData != null && entry.getKey().compareTo(previousData.getKey()) <= 0 ) { + //TODO : this is almost useless in case of reverse-sortered query because everything will be deleted by the Syncer before asking the second item + logger.error(src.getDataSourceName() + " : Input data is not well ordered but the sync task require it : '" + + entry.getKey() + "' is not lexicographically greater than '" + previousData.getKey() + "'"); + // Escape the while loop + abort=true; continue; + } + + // We have found a valid entry, so escape gracefully the loop + done=true; + } else { + // Log read problems and choose between skip or abort + if ( ! this.skipReadErrors ) { + logger.error(src.getDataSourceName() + " has returned an invalid entry after " + hint); + // Escape the while loop + abort=true; continue; + } + if ( !alreadyWarned ) { + alreadyWarned=true; + logger.info("Invalid entry read but skipReadErrors is enabled, will try to read next entry (warned only once)"); + } + + // We don't have a valid entry, give a chance to the next iteration + done=false; + } /* if ( valid )*/ + + } /* while */ + + // If we don't have found anything valid, throw exception (better semantics than returning null) + if (!done) { + throw new NoSuchElementException(); + } + + // Keep track of previous read record + // -> for hinting in log messages when bad things happens + // -> to check if entries are well ordered + previousData=entry; + return entry; + } +} diff --git a/src/main/src/sync/BasicSyncTask.java b/src/main/src/sync/BasicSyncTask.java new file mode 100644 index 0000000..24f34a8 --- /dev/null +++ b/src/main/src/sync/BasicSyncTask.java @@ -0,0 +1,292 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package sync; + + +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Set; + +import org.apache.log4j.Logger; + +import data.MVDataEntry; +import data.io.MVDataReader; +import data.io.MVDataWriter; + +/** + * Basic one-way synchronization code. Uses MVDataEntry semantics. + * Each entry has a key and a set of multi-valued attributes, like LDAP entries. + * Data source is a MVDataReader. Multiple source could be used via MVDataCombiner. + * <br/><br/> + * <b>Warnings :</b> needs MVDataReaders that give key-sorted results. This sync will try + * to delete entries that exists on destination side and don't exist at source side. + * Extra attributes in existing entries on destination side are preserved. + * Look like useful for account's failure password count for instance. + * <br/><br/> + * <b>Notes :</b> Null value and empty strings are not allowed in MVDataEntry, so they are not sync'ed. + * + * @author lpouzenc + */ +public class BasicSyncTask extends AbstractSyncTask { + private static final Logger logger = Logger.getLogger(BasicSyncTask.class.getName()); + + /** + * Source data stream (read-only) + */ + private final MVDataReader srcReader; + /** + * Destination data stream (read) + */ + private final MVDataReader dstReader; + /** + * Destination data stream (write) + */ + private final MVDataWriter dstWriter; + + /** + * If true, disable removal of data on destination side even if detected as obsolete + */ + private boolean skipEntryDelete; + + + private int maxInserts; + private int maxUpdates; + private int maxDeletes; + + private transient int curInserts; + private transient int curUpdates; + private transient int curDeletes; + + + /** + * BasicSyncTask constructor + * Assumes that the *Readers have iterators that returns entries sorted by lexicographical ascending key + * @param taskName Friendly name of this task (for tracing in log files) + * @param srcReader Source data stream (read-only) + * @param dstReader Destination data stream (read) + * @param dstWriter Destination data stream (write) + */ + public BasicSyncTask(String taskName, boolean skipDelete, MVDataReader srcReader, MVDataReader dstReader, MVDataWriter dstWriter) { + this.taskName = taskName; + this.srcReader = srcReader; + this.dstReader = dstReader; + this.dstWriter = dstWriter; + + this.maxInserts = 0; + this.maxUpdates = 0; + this.maxDeletes = 0; + } + + public Boolean call() { + logger.info("task " + taskName + " : starting " + (dryRun?"dry-run":"real") + " pass"); + // Better stack traces "call()" don't say "what" + boolean success = syncTaskRun(); + logger.info("task " + taskName + " : " + (success?"terminated successfully":"aborted")); + + return success; + } + + private boolean syncTaskRun() { + curInserts=0; + curUpdates=0; + curDeletes=0; + dstWriter.setDryRun(dryRun); + + Iterator<MVDataEntry> itSrc = srcReader.iterator(); + Iterator<MVDataEntry> itDst = dstReader.iterator(); + MVDataEntry src = null, dst = null; + boolean srcExhausted = false; + boolean dstExhausted = false; + boolean abort = false; + boolean done = false; + while ( !abort && !done ) { + + // Look-ahead srcReader if previous has been "poped" (or if never read yet) + if ( src == null ) { + if ( !srcExhausted ) { + srcExhausted = !itSrc.hasNext(); + } + if ( !srcExhausted ) { + try { + src=itSrc.next(); + logger.trace("src read : " + src); + } catch (Exception e) { + logger.error("Read failure detected on " + srcReader.getDataSourceName() + ". Aborting.", e); + // Escape from the while loop + abort=true; continue; + } + } + } + + // Look-ahead dstReader if previous has been "poped" (or if never read yet) + if ( dst == null ) { + if ( !dstExhausted ) { + dstExhausted = !itDst.hasNext(); + } + if ( !dstExhausted ) { + try { + dst = itDst.next(); + logger.trace("dst read : " + dst); + } catch (NoSuchElementException e) { + logger.error("Read failure detected on " + dstReader.getDataSourceName() + ". Aborting.", e); + // Escape from the while loop + abort=true; continue; + } + } + } + + // Error-free cases (no problems while reading data) + int compare; + if ( !srcExhausted && !dstExhausted ) { + // General case : check order precedence to take an action + compare = src.compareTo(dst); + } else if ( !srcExhausted && dstExhausted ) { + // Particular case : dst is exhausted, it's like ( src < dst ) + compare=-1; + } else if ( srcExhausted && !dstExhausted ) { + // Particular case : src is exhausted, it's like ( src > dst ) + compare=1; + } else /* ( srcExhausted && dstExhausted ) */ { + // Particular case : everything is synchronized + // Exit gracefully the while loop + done=true; continue; + } + + logger.trace("compare : " + compare); + + boolean actionRealized = false; + // Take an action (insert/update/delete) + if ( compare < 0 ) { + actionRealized = _insert(src); + src = null; + // preserve dst until src key is not greater + } else if ( compare > 0 ) { + // dst current entry doesn't exists anymore (src key is greater than dst key) + actionRealized = _delete(dst); + // preserve src until dst key is not greater + dst = null; + } else /* ( compare == 0 ) */ { + // src current entry already exists in dst, update it if necessary + Set<String> changedAttr = src.getChangedAttributes(dst); + if ( ! changedAttr.isEmpty() ) { + actionRealized = _update(src,dst,changedAttr); + } else { + // Already up-to-date, nothing to do + actionRealized = true; + } + // Both src and dst have been used + src = null; + dst = null; + } + abort = !actionRealized; + } /* while */ + + return !abort; + } /* _taskRunSync() */ + + private boolean _insert(MVDataEntry entry) { + + if ( maxInserts > 0 && curInserts >= maxInserts ) { + logger.error("Max insert limit reached (" + maxInserts + ")" ); + return false; + } + + logger.debug("dstWriter : Action\n-> Insert " + entry); + try { + dstWriter.insert(entry); + } catch (Exception e) { + logger.error("Exception occured while inserting", e); + return false; + } + + curInserts++; + return true; + } + + private boolean _update(MVDataEntry updatedEntry, MVDataEntry originalEntry, Set<String> attrToUpdate) { + if ( maxUpdates > 0 && curUpdates >= maxUpdates ) { + logger.error("Max update limit reached (" + maxUpdates + ")"); + return false; + } + + logger.debug("dstWriter : Action\n-> Update " + updatedEntry + "\n-> changed attributes : " + attrToUpdate); + try { + dstWriter.update(updatedEntry, originalEntry, attrToUpdate); + } catch (Exception e) { + logger.error("Exception occured while updating", e); + return false; + } + + curUpdates++; + return true; + } + + private boolean _delete(MVDataEntry entry) { + if ( skipEntryDelete ) { + logger.info("dstWriter : skipping deletion for key " + entry.getKey()); + return true; + } + + if ( maxDeletes > 0 && curDeletes >= maxDeletes ) { + logger.error("Max delete limit reached (" + maxDeletes + ")"); + return false; + } + logger.debug("dstWriter : Action\n-> Delete " + entry); + try { + dstWriter.delete(entry); + } catch (Exception e) { + logger.error("Exception occured while deleting", e); + return false; + } + + curDeletes++; + return true; + } + + // Boring accessors + + /** + * Setter to fix limits about operations counts (safeguard) + * @param maxInserts + * @param maxUpdates + * @param maxDeletes + */ + public void setOperationLimits(int maxInserts, int maxUpdates, int maxDeletes) { + this.maxInserts = maxInserts; + this.maxUpdates = maxUpdates; + this.maxDeletes = maxDeletes; + } + + /** + * @return the skipEntryDelete + */ + public boolean isSkipEntryDelete() { + return skipEntryDelete; + } + + /** + * @param skipEntryDelete the skipEntryDelete to set + */ + public void setSkipEntryDelete(boolean skipEntryDelete) { + this.skipEntryDelete = skipEntryDelete; + } + +} diff --git a/src/main/src/utils/JVMStatsDumper.java b/src/main/src/utils/JVMStatsDumper.java new file mode 100644 index 0000000..41f1d97 --- /dev/null +++ b/src/main/src/utils/JVMStatsDumper.java @@ -0,0 +1,111 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package utils; + +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.MemoryUsage; +import java.lang.management.RuntimeMXBean; + +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +/** + * TODO javadoc + * + * @author lpouzenc + */ +public class JVMStatsDumper { + private static final Logger logger = Logger.getLogger(JVMStatsDumper.class.getName()); + + public static void logGCStats() { + // Skip all string construction if will not print this stuff + if ( logger.getLevel().isGreaterOrEqual(Level.INFO) ) { return; } + + long totalGarbageCollections = 0; + long garbageCollectionTime = 0; + + final String gcDumpHeader="Dumping Garbage Collector statistics\n" + + "+--------------------+-----------------------------+\n" + + "+ GC Name + Count + Time (ms) +\n" + + "+--------------------+--------------+--------------+\n"; + + StringBuilder sb = new StringBuilder(1024); + sb.append(gcDumpHeader); + + for(GarbageCollectorMXBean gc : ManagementFactory.getGarbageCollectorMXBeans()) { + + long count = gc.getCollectionCount(); + long time = gc.getCollectionTime(); + + sb.append(String.format("+ %18s + %,12d + %,12d +%n", gc.getName(), count, time)); + + if(count >= 0) totalGarbageCollections += count; + if(time >= 0) garbageCollectionTime += time; + } + + sb.append("+ + + +\n"); + sb.append(String.format("+ %18s + %,12d + %,12d +%n", + "Total", totalGarbageCollections, garbageCollectionTime + )); + sb.append("+--------------------+--------------+--------------+\n"); + + sb.append("JVM arguments : "); + RuntimeMXBean runtimeMxBean = ManagementFactory.getRuntimeMXBean(); + for ( String arg : runtimeMxBean.getInputArguments() ) { + sb.append(arg + " "); + } + + logger.debug(sb); + } + + /** + * Helper function to log the current memory usage + */ + public static void logMemoryUsage() { + // Skip all string construction if will not print this stuff + if ( logger.getLevel().isGreaterOrEqual(Level.INFO) ) { return; } + + final String memDumpHeader="Dumping memory statistics\n" + + "+--------------------------------------------------------------------------------+\n" + + "+ + Current (kio) + Peak (kio) +\n" + + "+ Pool +-----------------------------------------------------------+\n" + + "+ + Used + Committed + Used + Committed +\n" + + "+--------------------+--------------+--------------+--------------+--------------+\n"; + + StringBuilder sb = new StringBuilder(1024); + sb.append(memDumpHeader); + + for (MemoryPoolMXBean pool : ManagementFactory.getMemoryPoolMXBeans()) { + MemoryUsage peak = pool.getPeakUsage(); + MemoryUsage curr = pool.getUsage(); + sb.append(String.format("+ %18s + %,12d + %,12d + %,12d + %,12d +%n", + pool.getName(),curr.getUsed()/1024, curr.getCommitted()/1024, peak.getUsed()/1024, peak.getCommitted()/1024 + )); + pool.resetPeakUsage(); //XXX Maybe this is not a global action and is useless on a temporary object used once + } + sb.append("+--------------------+--------------+--------------+--------------+--------------+\n"); + + logger.debug(sb); + } + +} diff --git a/src/main/sssync.sh b/src/main/sssync.sh new file mode 100755 index 0000000..43a1810 --- /dev/null +++ b/src/main/sssync.sh @@ -0,0 +1,3 @@ +#!/bin/sh +cd $(dirname $0) +java -jar SSSync.jar diff --git a/src/sloc.sh b/src/sloc.sh new file mode 100755 index 0000000..6b5cb02 --- /dev/null +++ b/src/sloc.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +mkdir /tmp/sloc/{,src,JUTests} +find */src -name *.java \! -name *Bean.java -print0 | xargs -0r cp -vt /tmp/sloc/src +find */JUTests -name *.java \! -name *Bean.java -print0 | xargs -0r cp -vt /tmp/sloc/JUTests +sloccount /tmp/sloc/* +rm -r /tmp/sloc |