Early development stages (before SCM) : WIP_1

Early development stages (before SCM) : WIP_2

Early development stages (before SCM) : WIP_3

Early development stages (before SCM) : WIP_4

Early development stages (before SCM) : WIP_6

Early development stages (before SCM) : WIP_7

Early development stages (before SCM) : WIP_8

Adds documentation folder as an Eclipse project.
Adds README for github.

Decent source tree by tuning Eclise project's location

One forgetten file while movign everything :)

Adding Copyright, licencing (GPL v3), correcting README

9 files changed, 461 insertions, 0 deletions

diff --git a/src/main/JUTests/AllClientServerTests.java b/src/main/JUTests/AllClientServerTests.java
new file mode 100644
index 0000000..cef8ffd
--- /dev/null
+++ b/src/main/JUTests/AllClientServerTests.java
@@ -0,0 +1,18 @@
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+import org.junit.runners.Suite.SuiteClasses;
+
+import data.io.ldap.LDAPDataReaderTest;
+import data.io.ldap.LDAPDataWriterTest;
+import data.io.sql.SQLRelDataReaderTest;
+
+
+@RunWith(Suite.class)
+@SuiteClasses({
+	// SSSync_Connectors
+	LDAPDataReaderTest.class, LDAPDataWriterTest.class,
+	SQLRelDataReaderTest.class,
+})
+public class AllClientServerTests {
+
+}
diff --git a/src/main/JUTests/AllLocalTests.java b/src/main/JUTests/AllLocalTests.java
new file mode 100644
index 0000000..bc9019d
--- /dev/null
+++ b/src/main/JUTests/AllLocalTests.java
@@ -0,0 +1,29 @@
+
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+import org.junit.runners.Suite.SuiteClasses;
+
+import sync.BasicSyncTaskTest;
+
+import conf.SSSyncConfParserTest;
+
+import data.MVDataEntryTest;
+import data.io.SafeDataReaderTest;
+import data.io.csv.CSVDataReaderTest;
+import data.io.filters.MVDataCombinerTest;
+
+@RunWith(Suite.class)
+@SuiteClasses( {
+	// SSSync
+	SSSyncConfParserTest.class,
+	SafeDataReaderTest.class,
+	BasicSyncTaskTest.class,
+	// SSSync_Connectors (only local)
+	CSVDataReaderTest.class,
+	// SSSync_Core
+	MVDataEntryTest.class, MVDataCombinerTest.class,
+	} )
+public class AllLocalTests {
+
+}
diff --git a/src/main/JUTests/conf/SSSyncConfParserTest.java b/src/main/JUTests/conf/SSSyncConfParserTest.java
new file mode 100644
index 0000000..100df16
--- /dev/null
+++ b/src/main/JUTests/conf/SSSyncConfParserTest.java
@@ -0,0 +1,69 @@
+package conf;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.net.URL;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.Yaml;
+
+public class SSSyncConfParserTest {
+	
+	private File currentFolder;
+	
+	@Before
+	public void setup() {
+		URL main = SSSyncConfParserTest.class.getResource("SSSyncConfParserTest.class");
+		if (!"file".equalsIgnoreCase(main.getProtocol()))
+			throw new IllegalStateException("This class is not stored in a file");
+		currentFolder = new File(main.getPath()).getParentFile();
+	}
+	
+	@Test
+	public void loadConfigTest() throws Exception {
+		
+		String expectedMain = readEntireFile(new File(currentFolder, "testExpectedMain.yaml"));
+		String expectedConn = readEntireFile(new File(currentFolder, "testExpectedConn.yaml"));
+		String mainConfigFile = new File(currentFolder, "testMain.yaml").getAbsolutePath();
+		String connConfigFile = new File(currentFolder, "testConn.yaml").getAbsolutePath();
+
+		// Loading (config => beans)
+		ConfigRootBean confMain = SSSyncConfParser.loadMainConfig(mainConfigFile);
+		ConfigConnectionsBean confConn = SSSyncConfParser.loadConnConfig(connConfigFile);
+		
+		
+		System.out.println(confMain);
+		System.out.println(confConn);
+		
+		// Dumping (beans => config)
+		DumperOptions options = new DumperOptions();
+		options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
+		Yaml yamlDump = new Yaml(options);
+		String dumpMain = yamlDump.dump(confMain);
+		String dumpConn = yamlDump.dump(confConn);
+		
+		// Checking that everything is kept
+		assertEquals(expectedMain, dumpMain);
+		assertEquals(expectedConn, dumpConn);
+	}
+	
+    private static String readEntireFile(File file) throws IOException {
+        FileReader in = new FileReader(file);
+        StringBuilder contents = new StringBuilder((int) file.length());
+        char[] buffer = new char[4096];
+        int read = 0;
+        do {
+            contents.append(buffer, 0, read);
+            read = in.read(buffer);
+        } while (read >= 0);
+        in.close();
+        
+        return contents.toString();
+    }
+
+}
diff --git a/src/main/JUTests/conf/testConn.yaml b/src/main/JUTests/conf/testConn.yaml
new file mode 100644
index 0000000..c41063c
--- /dev/null
+++ b/src/main/JUTests/conf/testConn.yaml
@@ -0,0 +1,19 @@
+# This file contains credentials (should be readable only by SSSync)
+connections:
+  - id  : ora_1
+    type: jdbc
+    dbms: oracle
+    ress: gest
+    host: ora.univ-jfc.fr
+    port: 1521
+    user: GRHUM
+    pass: secret
+    db  : GHRUM
+
+  - id  : ldap_1
+    type: ldap
+    host: localhost
+    port: 389
+    bind: uid=ldapadmin,ou=specialUsers,dc=univ-jfc,dc=fr
+    pass: secret
+
diff --git a/src/main/JUTests/conf/testExpectedConn.yaml b/src/main/JUTests/conf/testExpectedConn.yaml
new file mode 100644
index 0000000..4cb3421
--- /dev/null
+++ b/src/main/JUTests/conf/testExpectedConn.yaml
@@ -0,0 +1,22 @@
+!!conf.ConfigConnectionsBean
+connections:
+- bind: null
+  db: GHRUM
+  dbms: oracle
+  host: ora.univ-jfc.fr
+  id: ora_1
+  pass: secret
+  port: 1521
+  ress: gest
+  type: jdbc
+  user: GRHUM
+- bind: uid=ldapadmin,ou=specialUsers,dc=univ-jfc,dc=fr
+  db: null
+  dbms: null
+  host: localhost
+  id: ldap_1
+  pass: secret
+  port: 389
+  ress: null
+  type: ldap
+  user: null
diff --git a/src/main/JUTests/conf/testExpectedMain.yaml b/src/main/JUTests/conf/testExpectedMain.yaml
new file mode 100644
index 0000000..dd00aef
--- /dev/null
+++ b/src/main/JUTests/conf/testExpectedMain.yaml
@@ -0,0 +1,70 @@
+!!conf.ConfigRootBean
+globals:
+  maxExecTime: 3
+tasks:
+- destination:
+    attr: uid
+    base: ou=people,dc=univ-jfc,dc=fr
+    conn: ldap_1
+    kind: ldap
+    mode: null
+    name: LDAP de test, ou=people
+    path: null
+    query: null
+  name: People sync
+  opLimits:
+    delete: 10
+    insert: 100
+    update: 10
+  skipEntryDelete: false
+  skipReadErrors: false
+  sources:
+  - attr: null
+    base: null
+    conn: ora_1
+    kind: sql
+    mode: PRIMARY_SOURCE
+    name: GHRUM, comptes et personnes
+    path: null
+    query: people.sql
+  - attr: null
+    base: null
+    conn: null
+    kind: csv
+    mode: MERGE_APPEND
+    name: CSV personnes additionnelles
+    path: people_append.csv
+    query: null
+  - attr: null
+    base: null
+    conn: null
+    kind: sorted_csv
+    mode: MERGE_REPLACE
+    name: CSV correctifs personnes
+    path: people_replace.csv
+    query: null
+- destination:
+    attr: supannEntiteAffectation
+    base: ou=structures,dc=univ-jfc,dc=fr
+    conn: ldap_1
+    kind: ldap
+    mode: null
+    name: LDAP de test, ou=structures
+    path: null
+    query: null
+  name: Structure sync
+  opLimits:
+    delete: 10
+    insert: 10
+    update: 10
+  skipEntryDelete: true
+  skipReadErrors: true
+  sources:
+  - attr: null
+    base: null
+    conn: ora_1
+    kind: sql
+    mode: PRIMARY_SOURCE
+    name: GHRUM, structures
+    path: null
+    query: structures.sql
diff --git a/src/main/JUTests/conf/testMain.yaml b/src/main/JUTests/conf/testMain.yaml
new file mode 100644
index 0000000..39350b2
--- /dev/null
+++ b/src/main/JUTests/conf/testMain.yaml
@@ -0,0 +1,54 @@
+# This YAML file describe all synchronization tasks, with their readers and writers
+
+globals:
+    maxExecTime: 3
+
+tasks:
+  - name: People sync
+    opLimits:
+        insert: 100
+        update: 10
+        delete: 10
+    sources:
+      - name: GHRUM, comptes et personnes
+        kind: sql
+        conn: ora_1
+        mode: PRIMARY_SOURCE
+        query: people.sql
+
+      - name: CSV personnes additionnelles
+        kind: csv
+        mode: MERGE_APPEND
+        path: people_append.csv
+
+      - name: CSV correctifs personnes
+        kind: sorted_csv
+        mode: MERGE_REPLACE
+        path: people_replace.csv
+
+    destination:
+        name: LDAP de test, ou=people
+        kind: ldap
+        conn: ldap_1
+        attr: uid
+        base: ou=people,dc=univ-jfc,dc=fr
+
+  - name: Structure sync
+    sources:
+      - name: GHRUM, structures
+        kind: sql
+        conn: ora_1
+        mode: PRIMARY_SOURCE
+        query: structures.sql
+    destination:
+        name: LDAP de test, ou=structures
+        kind: ldap
+        conn: ldap_1
+        attr: supannEntiteAffectation
+        base: ou=structures,dc=univ-jfc,dc=fr
+    skipEntryDelete: true
+    skipReadErrors: true
+    opLimits:
+        insert: 10
+        update: 10
+        delete: 10
\ No newline at end of file
diff --git a/src/main/JUTests/data/io/SafeDataReaderTest.java b/src/main/JUTests/data/io/SafeDataReaderTest.java
new file mode 100644
index 0000000..427004b
--- /dev/null
+++ b/src/main/JUTests/data/io/SafeDataReaderTest.java
@@ -0,0 +1,51 @@
+package data.io;
+
+import static org.junit.Assert.*;
+
+import java.util.Iterator;
+
+import org.apache.log4j.PropertyConfigurator;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import data.MVDataEntry;
+import data.io.stub.StubDataReader;
+
+public class SafeDataReaderTest {
+	
+	private static final String LOG_PROPERTIES_FILE = "conf/log4j.properties";
+	
+	@BeforeClass
+	public static void setup() {
+		PropertyConfigurator.configure(LOG_PROPERTIES_FILE);
+	}
+
+	@Test
+	public void testNoErrors() {
+		MVDataEntry testEntries[] = new MVDataEntry[5];
+		for (int i=0;i<5;i++) {
+			testEntries[i] = new MVDataEntry("line"+(i+1));
+			testEntries[i].put("attr1", "value"+(i+1));
+		}		
+		
+		StubDataReader src = new StubDataReader("testNoSkipErrors_src", testEntries);
+		StubDataReader expected = new StubDataReader("testNoSkipErrors_expected", testEntries);
+		
+		SafeDataReader reader = new SafeDataReader(src, false);
+		
+		// Test twice to check if asking a new iterator "rewinds" correctly
+		for (int i=0;i<2;i++) {
+			//System.out.println("Loop " + (i+1));
+			Iterator<MVDataEntry> readerIt = reader.iterator();
+			for ( MVDataEntry e: expected) {
+				assertTrue(readerIt.hasNext());
+				MVDataEntry r = readerIt.next();
+				//System.out.println(e + " / " + r);
+				assertEquals(e, r);
+			}
+			assertFalse(readerIt.hasNext());
+		}
+	}
+
+	//TODO Real tests with messy input readers (null values, exception, hasNext/next() incoherence)
+}
diff --git a/src/main/JUTests/sync/BasicSyncTaskTest.java b/src/main/JUTests/sync/BasicSyncTaskTest.java
new file mode 100644
index 0000000..88d9c98
--- /dev/null
+++ b/src/main/JUTests/sync/BasicSyncTaskTest.java
@@ -0,0 +1,129 @@
+package sync;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+import org.apache.log4j.PropertyConfigurator;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import data.MVDataEntry;
+import data.filters.MVDataCombiner;
+import data.filters.MVDataCombiner.MVDataCombineMode;
+import data.io.MVDataReader;
+import data.io.SafeDataReader;
+import data.io.csv.CSVDataReader;
+import data.io.stub.StubDataReader;
+import data.io.stub.StubDataWriter;
+
+public class BasicSyncTaskTest {
+	
+	private static final String LOG_PROPERTIES_FILE = "conf/log4j.properties";
+	
+	@BeforeClass
+	public static void setup() {
+		PropertyConfigurator.configure(LOG_PROPERTIES_FILE);
+	}
+
+	@Test
+	public void test() throws IOException {
+		
+		// Input flows setup
+		MVDataEntry[] fakeEntries1 = new MVDataEntry[5];
+		fakeEntries1[0] = new MVDataEntry("line1");
+		fakeEntries1[0].put("hello", "world");
+		
+		fakeEntries1[1] = new MVDataEntry("line2");
+		fakeEntries1[1].put("bla", "hidden");
+		fakeEntries1[1].put("hello", "merged");
+		
+		fakeEntries1[2] = new MVDataEntry("line3");
+		fakeEntries1[2].put("hello", "world");
+		
+		fakeEntries1[3] = new MVDataEntry("line4");
+		fakeEntries1[3].put("hello", "world");
+		
+		fakeEntries1[4] = new MVDataEntry("line5");
+		fakeEntries1[4].put("hello", "world");		
+		
+		
+		MVDataEntry[] fakeEntries2 = new MVDataEntry[3];
+		fakeEntries2[0] = new MVDataEntry("line1");
+		fakeEntries2[0].put("hello", "world");
+		
+		fakeEntries2[1] = new MVDataEntry("line2");
+		fakeEntries2[1].put("bla", "replaced");
+		
+		fakeEntries2[2] = new MVDataEntry("line3");
+		fakeEntries2[2].put("hello", "world");
+		
+		
+		MVDataEntry[] fakeEntries3 = new MVDataEntry[5];
+		fakeEntries3[0] = new MVDataEntry("line2");
+		fakeEntries3[0].put("hello", "world");
+		fakeEntries3[0].put("extra", "to be preserved");
+
+		fakeEntries3[1] = new MVDataEntry("line2b");
+		fakeEntries3[1].put("to be", "removed", null);
+
+		fakeEntries3[2] = new MVDataEntry("line4");
+		fakeEntries3[2].put("hello", "world");
+		fakeEntries3[2].put("extra", "to be preserved");
+		
+		fakeEntries3[3] = new MVDataEntry("line5");
+		fakeEntries3[3].splitAndPut("hello", "too;much;world", ";");
+		
+		fakeEntries3[4] = new MVDataEntry("line6");
+		fakeEntries3[4].put("to be", "removed");
+		
+		StubDataReader fakeReader1 = new StubDataReader("testSrc1", fakeEntries1);
+		StubDataReader fakeReader2 = new StubDataReader("testSrc3", fakeEntries2);
+		StubDataReader fakeReader3 = new StubDataReader("testDst", fakeEntries3);
+		
+		MVDataReader readers[] = new MVDataReader[]{
+				new SafeDataReader(fakeReader1,false),
+				new SafeDataReader(
+					new CSVDataReader("testSrc2",
+						new StringReader(CSVDataReader.CSV_DEMO),
+						false
+					), false
+				),
+				new SafeDataReader(fakeReader2,false),
+		};
+				
+		MVDataCombineMode mergeModes[] = new MVDataCombineMode[]{
+				MVDataCombineMode.PRIMARY_SOURCE,
+				MVDataCombineMode.MERGE_APPEND,
+				MVDataCombineMode.MERGE_REPLACE,
+		};
+		
+		MVDataReader srcReader = new MVDataCombiner("testSrcComb", readers, mergeModes);
+		MVDataReader dstReader = fakeReader3;
+		
+		// Output flow setup
+		StubDataWriter dstWriter = new StubDataWriter(10);
+		
+		// Data sync'er initialization
+		BasicSyncTask task = new BasicSyncTask("task1", false, srcReader, dstReader, dstWriter);
+		task.setOperationLimits(10,10,10);
+		
+		// Data sync'er run
+		assertTrue(task.call());
+		
+		// Expected outputs
+		String expectedDstOps = 
+				"INSERT: {key=line1, attrValPairs={hello=[world], attr2=[csv1], from=[csv1, csv1bis]}}\n" +
+				"UPDATE: {key=line2, attrValPairs={hello=[the, merged, world, all], bla=[replaced]}}\n" +
+				"DELETE: {key=line2b, attrValPairs={to be=[removed]}}\n" +
+				"INSERT: {key=line3, attrValPairs={hello=[world]}}\n" +
+				// Line 4 must not be updated !
+				"UPDATE: {key=line5, attrValPairs={hello=[world]}}\n" +
+				"DELETE: {key=line6, attrValPairs={to be=[removed]}}\n";
+
+		// Check results
+		assertEquals(expectedDstOps, dstWriter.toString());
+	}
+
+}