diff options
Diffstat (limited to 'src/connectors')
24 files changed, 1463 insertions, 0 deletions
diff --git a/src/connectors/.classpath b/src/connectors/.classpath new file mode 100644 index 0000000..e421f8f --- /dev/null +++ b/src/connectors/.classpath @@ -0,0 +1,21 @@ +<?xml version="1.0" encoding="UTF-8"?> +<classpath> + <classpathentry kind="src" path="src"/> + <classpathentry kind="src" path="JUTests"/> + <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/> + <classpathentry combineaccessrules="false" kind="src" path="/SSSync_Core"/> + <classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/> + <classpathentry kind="lib" path="lib/commons-csv-1.0-SNAPSHOT.jar"/> + <classpathentry kind="lib" path="lib/ojdbc6.jar"> + <attributes> + <attribute name="javadoc_location" value="jar:platform:/resource/SSSync_Connectors/lib/ojdbc6-javadoc.jar!/"/> + </attributes> + </classpathentry> + <classpathentry kind="lib" path="lib/mysql-connector-java-5.1.31-bin.jar"/> + <classpathentry kind="lib" path="lib/unboundid-ldapsdk-se.jar"> + <attributes> + <attribute name="javadoc_location" value="jar:platform:/resource/SSSync_Connectors/lib/unboundid-ldapsdk-se-javadoc.jar!/"/> + </attributes> + </classpathentry> + <classpathentry kind="output" path="bin"/> +</classpath> diff --git a/src/connectors/.project b/src/connectors/.project new file mode 100644 index 0000000..b4f50df --- /dev/null +++ b/src/connectors/.project @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="UTF-8"?> +<projectDescription> + <name>SSSync_Connectors</name> + <comment></comment> + <projects> + </projects> + <buildSpec> + <buildCommand> + <name>org.eclipse.jdt.core.javabuilder</name> + <arguments> + </arguments> + </buildCommand> + </buildSpec> + <natures> + <nature>org.eclipse.jdt.core.javanature</nature> + </natures> +</projectDescription> diff --git a/src/connectors/.settings/org.eclipse.jdt.core.prefs b/src/connectors/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..8000cd6 --- /dev/null +++ b/src/connectors/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,11 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=1.6 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.source=1.6 diff --git a/src/connectors/JUTests/data/io/csv/CSVDataReaderTest.java b/src/connectors/JUTests/data/io/csv/CSVDataReaderTest.java new file mode 100644 index 0000000..6a0e053 --- /dev/null +++ b/src/connectors/JUTests/data/io/csv/CSVDataReaderTest.java @@ -0,0 +1,50 @@ +package data.io.csv; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Iterator; + +import org.junit.Test; + +import data.MVDataEntry; + +public class CSVDataReaderTest { + + + @Test + public void testNext() throws IOException { + CSVDataReader reader = new CSVDataReader( + "testNext", + new StringReader(CSVDataReader.CSV_DEMO), + false + ); + + MVDataEntry expected[] = new MVDataEntry[3]; + expected[0]=new MVDataEntry("line1"); + expected[0].splitAndPut("from", "csv1;csv1bis", ";"); + expected[0].splitAndPut("attr2","csv1",";"); + + expected[1]=new MVDataEntry("line2"); + expected[1].splitAndPut("hello", "all;the;world", ";"); + + expected[2]=new MVDataEntry("line3"); + expected[2].splitAndPut("hello", "all;the;others", ";"); + + // Test twice to check if asking a new iterator "rewinds" correctly + for (int i=0;i<2;i++) { + System.out.println("Loop " + (i+1)); + Iterator<MVDataEntry> readerIt = reader.iterator(); + + for ( MVDataEntry e: expected) { + assertTrue(readerIt.hasNext()); + MVDataEntry r = readerIt.next(); + System.out.println(e + " / " + r); + assertEquals(e, r); + } + assertFalse(readerIt.hasNext()); + } + } + +} diff --git a/src/connectors/JUTests/data/io/ldap/LDAPDataReaderTest.java b/src/connectors/JUTests/data/io/ldap/LDAPDataReaderTest.java new file mode 100644 index 0000000..dcfc602 --- /dev/null +++ b/src/connectors/JUTests/data/io/ldap/LDAPDataReaderTest.java @@ -0,0 +1,94 @@ +package data.io.ldap; + +import static org.junit.Assert.assertTrue; + +import org.junit.Before; +import org.junit.Test; + +import data.MVDataEntry; + +public class LDAPDataReaderTest { + + LDAPConnectionWrapper builder; + + @Before + public void setup() { + builder = new LDAPConnectionWrapper("localhost", 389, "uid=ldapadmin,ou=specialUsers,dc=univ-jfc,dc=fr", "secret"); + } + + /* + @Test + public void testLookAhead1() { + _testLookAhead(1); + } + */ + + @Test + public void testLookAhead16() { + _testLookAhead(16); + } + + @Test + public void testLookAhead32() { + _testLookAhead(32); + } + + @Test + public void testLookAhead64() { + _testLookAhead(64); + } + + @Test + public void testLookAhead128() { + _testLookAhead(128); + } + + @Test + public void testLookAhead192() { + _testLookAhead(192); + } + + @Test + public void testLookAhead256() { + _testLookAhead(256); + } + + @Test + public void testLookAhead512() { + _testLookAhead(512); + } + + @Test + public void testLookAhead1024() { + _testLookAhead(1024); + } + + private void _testLookAhead(int lookAheadAmount) { + System.out.println("_testLookAhead("+lookAheadAmount+")"); + LDAPFlatDataReader reader = builder.newFlatReader("ldap_test", "ou=people,dc=univ-jfc,dc=fr", "uid", lookAheadAmount); + + int resultCount = 0; + String previousKey=null; + for ( MVDataEntry entry : reader ) { + //System.out.println(entry); + if ( previousKey != null ) assertTrue(entry.getKey().compareTo(previousKey) > 0); + resultCount++; + previousKey=entry.getKey(); + } + System.out.println(resultCount); + assertTrue(resultCount>0); + + // Second time with a second iterator (must give the same results) + int newResultCount = 0; + previousKey=null; + for ( MVDataEntry entry : reader ) { + //System.out.println(entry); + if ( previousKey != null ) assertTrue(entry.getKey().compareTo(previousKey) > 0); + newResultCount++; + previousKey=entry.getKey(); + } + System.out.println(newResultCount); + assertTrue(newResultCount == resultCount); + + } +} diff --git a/src/connectors/JUTests/data/io/ldap/LDAPDataWriterTest.java b/src/connectors/JUTests/data/io/ldap/LDAPDataWriterTest.java new file mode 100644 index 0000000..01a8af0 --- /dev/null +++ b/src/connectors/JUTests/data/io/ldap/LDAPDataWriterTest.java @@ -0,0 +1,16 @@ +package data.io.ldap; + +import static org.junit.Assert.*; + +import org.junit.Test; + +public class LDAPDataWriterTest { + + @Test + public void test() { + fail("Not yet implemented"); + } + + // TODO : test update() extensively : null, empty string, add/update/delete subcases... + +} diff --git a/src/connectors/JUTests/data/io/sql/SQLRelDataReaderTest.java b/src/connectors/JUTests/data/io/sql/SQLRelDataReaderTest.java new file mode 100644 index 0000000..a97a98d --- /dev/null +++ b/src/connectors/JUTests/data/io/sql/SQLRelDataReaderTest.java @@ -0,0 +1,115 @@ +package data.io.sql; + +import static org.junit.Assert.*; + +import java.io.File; +import java.io.IOException; +import java.net.URL; + +import org.junit.Before; +import org.junit.Test; + +import data.MVDataEntry; +import data.io.MVDataReader; +import data.io.sql.SQLConnectionWrapper.DBMSType; + + +/* + +CREATE TABLE sssync.people ( + uid CHAR(16) NULL , + uidNumber INT NOT NULL , + gidNumber INT NULL , + cn VARCHAR(45) NULL , + sn VARCHAR(45) NULL , + homeDirectory VARCHAR(45) NULL , + PRIMARY KEY (uid) ); +INSERT INTO sssync.people (uid, uidNumber, gidNumber, cn, sn, homeDirectory) VALUES ('lpouzenc', 1000, 999, 'Ludovic', 'Pouzenc', '/home/lpouzenc'); +INSERT INTO sssync.people (uid, uidNumber, gidNumber, cn, sn, homeDirectory) VALUES ('dpouzenc', 1001, 999, 'Daniel', 'Pouzenc', '/home/dpouzenc'); + + +for i in $(seq 10000 20000); do echo "INSERT INTO sssync.people (uid, uidNumber, gidNumber, cn, sn, homeDirectory) VALUES ('test$i', $i, 999, '$i', 'test', '/home/test$i');"; done | mysql -uroot -p + + + +DROP TABLE IF EXISTS structures; +CREATE TABLE structures ( + supannCodeEntite varchar(15) NOT NULL, + ou varchar(45) NOT NULL, + supannTypeEntite varchar(45) NOT NULL, + supannCodeEntiteParent varchar(45) NOT NULL, + PRIMARY KEY (supannCodeEntite) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +INSERT INTO structures VALUES ('2','CUFR','Etablissement','2'),('9','Personnels','Groupe','2'); + + + +TODO : make automated tests with embded Derby base + + */ +public class SQLRelDataReaderTest { + + private static final String TEST_REQUEST = "SELECT p.*, \"person;posixAccount;top\" as objectClass" + + " FROM sssync.people p" + + " ORDER BY 1 ASC;"; + + private SQLConnectionWrapper builder; + private MVDataReader reader1; + private MVDataReader reader2; + + @Before + public void setup() throws IOException { + // Find the folder containing this test class + URL main = SQLRelDataReaderTest.class.getResource("SQLRelDataReaderTest.class"); + if (!"file".equalsIgnoreCase(main.getProtocol())) + throw new IllegalStateException("This class is not stored in a file"); + File currentFolder = new File(main.getPath()).getParentFile(); + + // Build a connection and two readers on it + builder = new SQLConnectionWrapper(DBMSType.mysql, "localhost", 3306, null, "root", "secret", "sssync"); + reader1 = builder.newReader("testMysql1", TEST_REQUEST); + reader2 = builder.newReader("testMysql2", new File(currentFolder, "req_test.sql")); + } + + @Test + public void testNext() { + // First full read on reader1 + int resultCount_r1i1 = 0; + String previousKey_r1i1=null; + for ( MVDataEntry entry : reader1 ) { + //System.out.println(entry); + if ( previousKey_r1i1 != null ) assertTrue(entry.getKey().compareTo(previousKey_r1i1) > 0); + resultCount_r1i1++; + previousKey_r1i1=entry.getKey(); + } + System.out.println(resultCount_r1i1); + assertTrue(resultCount_r1i1 > 0); + + // First half read on reader2 + int resultCount_r2i1 = 0; + String previousKey_r2i1=null; + for ( MVDataEntry entry : reader2 ) { + //System.out.println(entry); + if ( previousKey_r2i1 != null ) assertTrue(entry.getKey().compareTo(previousKey_r2i1) > 0); + resultCount_r2i1++; + previousKey_r2i1=entry.getKey(); + if ( resultCount_r2i1 > resultCount_r1i1 / 2 ) break; + } + System.out.println(resultCount_r2i1); + assertTrue(resultCount_r2i1 > resultCount_r1i1 / 2 ); + + // Second time with a second iterator on reader1 (must give the same results than r1i1) + int resultCount_r1i2 = 0; + String previousKey_r1i2=null; + for ( MVDataEntry entry : reader1 ) { + //System.out.println(entry); + if ( previousKey_r1i2 != null ) assertTrue(entry.getKey().compareTo(previousKey_r1i2) > 0); + resultCount_r1i2++; + previousKey_r1i2=entry.getKey(); + } + System.out.println(resultCount_r1i2); + assertTrue(resultCount_r1i2 == resultCount_r1i1); + } + +} diff --git a/src/connectors/JUTests/data/io/sql/req_test.sql b/src/connectors/JUTests/data/io/sql/req_test.sql new file mode 100644 index 0000000..ab66d5f --- /dev/null +++ b/src/connectors/JUTests/data/io/sql/req_test.sql @@ -0,0 +1,5 @@ +SELECT + p.*, + "person;posixAccount;top" as objectClass +FROM sssync.people p +ORDER BY 1 ASC; diff --git a/src/connectors/build.xml b/src/connectors/build.xml new file mode 100644 index 0000000..fdae9de --- /dev/null +++ b/src/connectors/build.xml @@ -0,0 +1,89 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- WARNING: Eclipse auto-generated file. + Any modifications will be overwritten. + To include a user specific buildfile here, simply create one in the same + directory with the processing instruction <?eclipse.ant.import?> + as the first entry and export the buildfile again. --> +<project basedir="." default="build" name="SSSync_Connectors"> + <property environment="env"/> + <property name="SSSync_Main.location" value="../main"/> + <property name="ECLIPSE_HOME" value="../../../../../../usr/lib/eclipse"/> + <property name="SSSync_Core.location" value="../core"/> + <property name="debuglevel" value="source,lines,vars"/> + <property name="target" value="1.6"/> + <property name="source" value="1.6"/> + <path id="JUnit 4.libraryclasspath"> + <pathelement location="../../../../../../usr/share/eclipse/dropins/jdt/plugins/org.junit_4.8.2.dist/junit.jar"/> + <pathelement location="../../../../../../usr/share/eclipse/dropins/jdt/plugins/org.hamcrest.core_1.1.0.jar"/> + </path> + <path id="SSSync_Core.classpath"> + <pathelement location="${SSSync_Core.location}/bin"/> + <pathelement location="${SSSync_Core.location}/lib/guava-16.0.1.jar"/> + <path refid="JUnit 4.libraryclasspath"/> + </path> + <path id="SSSync_Connectors.classpath"> + <pathelement location="bin"/> + <path refid="SSSync_Core.classpath"/> + <path refid="JUnit 4.libraryclasspath"/> + <pathelement location="lib/commons-csv-1.0-SNAPSHOT.jar"/> + <pathelement location="lib/ojdbc6.jar"/> + <pathelement location="lib/mysql-connector-java-5.1.31-bin.jar"/> + <pathelement location="lib/unboundid-ldapsdk-se.jar"/> + </path> + <target name="init"> + <mkdir dir="bin"/> + <copy includeemptydirs="false" todir="bin"> + <fileset dir="src"> + <exclude name="**/*.java"/> + </fileset> + </copy> + <copy includeemptydirs="false" todir="bin"> + <fileset dir="JUTests"> + <exclude name="**/*.java"/> + </fileset> + </copy> + </target> + <target name="clean"> + <delete dir="bin"/> + </target> + <target depends="clean" name="cleanall"> + <ant antfile="build.xml" dir="${SSSync_Core.location}" inheritAll="false" target="clean"/> + </target> + <target depends="build-subprojects,build-project" name="build"/> + <target name="build-subprojects"> + <ant antfile="build.xml" dir="${SSSync_Core.location}" inheritAll="false" target="build-project"> + <propertyset> + <propertyref name="build.compiler"/> + </propertyset> + </ant> + </target> + <target depends="init" name="build-project"> + <echo message="${ant.project.name}: ${ant.file}"/> + <javac debug="true" debuglevel="${debuglevel}" destdir="bin" includeantruntime="false" source="${source}" target="${target}"> + <src path="src"/> + <src path="JUTests"/> + <classpath refid="SSSync_Connectors.classpath"/> + </javac> + </target> + <target description="Build all projects which reference this project. Useful to propagate changes." name="build-refprojects"> + <ant antfile="build.xml" dir="${SSSync_Main.location}" inheritAll="false" target="clean"/> + <ant antfile="build.xml" dir="${SSSync_Main.location}" inheritAll="false" target="build"> + <propertyset> + <propertyref name="build.compiler"/> + </propertyset> + </ant> + </target> + <target description="copy Eclipse compiler jars to ant lib directory" name="init-eclipse-compiler"> + <copy todir="${ant.library.dir}"> + <fileset dir="${ECLIPSE_HOME}/plugins" includes="org.eclipse.jdt.core_*.jar"/> + </copy> + <unzip dest="${ant.library.dir}"> + <patternset includes="jdtCompilerAdapter.jar"/> + <fileset dir="${ECLIPSE_HOME}/plugins" includes="org.eclipse.jdt.core_*.jar"/> + </unzip> + </target> + <target description="compile project with Eclipse compiler" name="build-eclipse-compiler"> + <property name="build.compiler" value="org.eclipse.jdt.core.JDTCompilerAdapter"/> + <antcall target="build"/> + </target> +</project> diff --git a/src/connectors/lib/commons-csv-1.0-SNAPSHOT.jar b/src/connectors/lib/commons-csv-1.0-SNAPSHOT.jar Binary files differnew file mode 100644 index 0000000..f6a74f1 --- /dev/null +++ b/src/connectors/lib/commons-csv-1.0-SNAPSHOT.jar diff --git a/src/connectors/lib/derby.jar b/src/connectors/lib/derby.jar Binary files differnew file mode 100644 index 0000000..a4d56f0 --- /dev/null +++ b/src/connectors/lib/derby.jar diff --git a/src/connectors/lib/derbytools.jar b/src/connectors/lib/derbytools.jar Binary files differnew file mode 100644 index 0000000..216ff3e --- /dev/null +++ b/src/connectors/lib/derbytools.jar diff --git a/src/connectors/lib/mysql-connector-java-5.1.31-bin.jar b/src/connectors/lib/mysql-connector-java-5.1.31-bin.jar Binary files differnew file mode 100644 index 0000000..85ae51d --- /dev/null +++ b/src/connectors/lib/mysql-connector-java-5.1.31-bin.jar diff --git a/src/connectors/lib/ojdbc6-javadoc.jar b/src/connectors/lib/ojdbc6-javadoc.jar Binary files differnew file mode 100644 index 0000000..81dfb08 --- /dev/null +++ b/src/connectors/lib/ojdbc6-javadoc.jar diff --git a/src/connectors/lib/ojdbc6.jar b/src/connectors/lib/ojdbc6.jar Binary files differnew file mode 100644 index 0000000..767eba7 --- /dev/null +++ b/src/connectors/lib/ojdbc6.jar diff --git a/src/connectors/lib/orai18n.jar b/src/connectors/lib/orai18n.jar Binary files differnew file mode 100644 index 0000000..9fad382 --- /dev/null +++ b/src/connectors/lib/orai18n.jar diff --git a/src/connectors/lib/unboundid-ldapsdk-se-javadoc.jar b/src/connectors/lib/unboundid-ldapsdk-se-javadoc.jar Binary files differnew file mode 100644 index 0000000..b724779 --- /dev/null +++ b/src/connectors/lib/unboundid-ldapsdk-se-javadoc.jar diff --git a/src/connectors/lib/unboundid-ldapsdk-se.jar b/src/connectors/lib/unboundid-ldapsdk-se.jar Binary files differnew file mode 100644 index 0000000..0932139 --- /dev/null +++ b/src/connectors/lib/unboundid-ldapsdk-se.jar diff --git a/src/connectors/src/data/io/csv/CSVDataReader.java b/src/connectors/src/data/io/csv/CSVDataReader.java new file mode 100644 index 0000000..6dbc8ff --- /dev/null +++ b/src/connectors/src/data/io/csv/CSVDataReader.java @@ -0,0 +1,248 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ +package data.io.csv; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; + +import data.MVDataEntry; +import data.io.AbstractMVDataReader; + +/** + * Stream-oriented reader from a particular CSV file. + * Always returns lines/items sorted by lexicographical ascending key. + * + * @author lpouzenc + */ +public class CSVDataReader extends AbstractMVDataReader { + + public static final String CSV_DEMO = + //"key,attr,values\n" + + "line3,hello,all;the;others\n" + + "line1,from,csv1;csv1bis\n" + + "line2,hello,all;the;world\n" + + "line1,attr2,csv1\n" + + ",,\n"; + + public static final CSVFormat DEFAULT_CSV_FORMAT = CSVFormat.EXCEL + .withHeader("key","attr","values") + .withIgnoreSurroundingSpaces(true); + + private final CSVFormat format; + private final Reader dataSourceStream; + + private transient MVDataEntry nextEntry; + private transient CSVRecord nextCSVRecord; + private transient Iterator<CSVRecord> csvIt; + + + /** + * Constructs a CSVDataReader object for parsing a CSV input given via dataSourceStream. + * @param dataSourceName A short string representing this reader (for logging) + * @param dataSourceStream A java.io.Reader from which read the actual CSV data, typically a FileReader + * @param alreadySorted If false, memory cost is around 3 times the CSV file size ! + * @param format Specify the exact format used to encode the CSV file (separators, escaping...) + * @throws IOException + */ + public CSVDataReader(String dataSourceName, Reader dataSourceStream, boolean alreadySorted, CSVFormat format) throws IOException { + this.dataSourceName = dataSourceName; + this.format = format; + + if ( alreadySorted ) { + this.dataSourceStream = dataSourceStream; + } else { + BufferedReader bufReader; + if ( dataSourceStream instanceof BufferedReader ) { + bufReader = (BufferedReader) dataSourceStream; + } else { + bufReader = new BufferedReader(dataSourceStream); + } + this.dataSourceStream = readAndSortLines(bufReader); + } + } + + /** + * Constructs a CSVDataReader object with default CSV format (for CSVParser). + * @param dataSourceName A short string representing this reader (for logging) + * @param dataSourceStream A java.io.Reader from which read the actual CSV data, typically a FileReader + * @param alreadySorted If false, memory cost is around 3 times the CSV file size ! + * @throws IOException + */ + public CSVDataReader(String dataSourceName, Reader dataSourceStream, boolean alreadySorted) throws IOException { + this(dataSourceName, dataSourceStream, alreadySorted, DEFAULT_CSV_FORMAT); + } + + /** + * {@inheritDoc} + * Note : multiple iterators on the same instance are not supported + */ + @Override + public Iterator<MVDataEntry> iterator() { + // When a new iterator is requested, everything should be reset + CSVParser parser; + try { + dataSourceStream.reset(); + parser = new CSVParser(dataSourceStream, format); + } catch (IOException e) { + throw new RuntimeException(e); + } + csvIt = parser.iterator(); + nextCSVRecord = null; + nextEntry = null; + return this; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean hasNext() { + if ( nextEntry == null ) { + lookAhead(); + } + return ( nextEntry != null ); + } + + /** + * {@inheritDoc} + */ + @Override + public MVDataEntry next() { + if ( !hasNext() ) { + throw new NoSuchElementException(); + } + // Pop the lookahead record + MVDataEntry res = nextEntry; + nextEntry=null; + // And return it + return res; + } + + /** + * In-memory File sorting, return as a single String + * @param reader + * @return + * @throws IOException + */ + private Reader readAndSortLines(BufferedReader bufReader) throws IOException { + // Put all the CSV in memory, in a SortedSet + SortedSet<String> lineSet = new TreeSet<String>(); + String inputLine; + int totalCSVSize=0; + while ((inputLine = bufReader.readLine()) != null) { + lineSet.add(inputLine); + totalCSVSize += inputLine.length() + 1; + } + bufReader.close(); // Closes also dataSourceStream + + // Put all sorted lines in a String + StringBuilder allLines = new StringBuilder(totalCSVSize); + for ( String line: lineSet) { + allLines.append(line + "\n"); + } + lineSet = null; // Could help the GC if the input file is huge + + // Build a Java Reader from that String + return new StringReader(allLines.toString()); + } + + /** + * A MVDataEntry could be represented on many CSV lines. + * The key is repeated, the attr could change, the values should change (for given key/attr pair) + */ + private void lookAhead() { + MVDataEntry currEntry = null; + + boolean abort=(nextCSVRecord==null && !csvIt.hasNext()); // Nothing to crunch + boolean done=(nextEntry!=null); // Already looked ahead + while (!abort && !done) { + // Try to get a valid CSVRecord + if ( nextCSVRecord == null ) { + nextCSVRecord = nextValidCSVRecord(); + } + // If no more CSV data + if ( nextCSVRecord == null ) { + // Maybe we have a remaining entry to return + if ( currEntry != null ) { + done=true; continue; + } else { + abort=true; continue; + } + } + + // Now we have a valid CSV line to put in a MVDataEntry + String newKey = nextCSVRecord.get("key"); + + + // If no MVDataEntry yet, it's time to create it (we have data to put into) + if ( currEntry == null ) { + currEntry = new MVDataEntry(newKey); + } + // If CSV line key matches MVDataEntry key, appends attr/values on it + // XXX Tricky code : following condition is always true if the previous one is true + if ( currEntry.getKey().equals(newKey) ) { + currEntry.splitAndPut(nextCSVRecord.get("attr"), nextCSVRecord.get("values"), ";"); + nextCSVRecord = null; // Record consumed + } else { + // Keys are different, we are done (and we have remaining CSV data in nextCSVRecord) + done=true; continue; + } + } + + nextEntry = done?currEntry:null; + } + + /** + * Seek for the next valid record in the CSV file + * @return the next valid CSVRecord + */ + private CSVRecord nextValidCSVRecord() { + CSVRecord res = null; + boolean abort = !csvIt.hasNext(); + boolean done = false; + while (!abort && !done) { + // Try to read a CSV line + res = (csvIt.hasNext())?csvIt.next():null; + + // Break if nothing readable + if ( res == null ) { + abort=true; continue; + } + + // Skip invalid and empty lines + String key = res.get("key"); + if ( key != null && ! key.isEmpty() ) { + done=true; continue; + } + } + + return done?res:null; + } +} diff --git a/src/connectors/src/data/io/ldap/LDAPConnectionWrapper.java b/src/connectors/src/data/io/ldap/LDAPConnectionWrapper.java new file mode 100644 index 0000000..3f6497b --- /dev/null +++ b/src/connectors/src/data/io/ldap/LDAPConnectionWrapper.java @@ -0,0 +1,112 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.io.ldap; + +import java.io.Closeable; +import java.io.IOException; + +import com.unboundid.ldap.sdk.BindResult; +import com.unboundid.ldap.sdk.LDAPConnection; +import com.unboundid.ldap.sdk.LDAPConnectionOptions; +import com.unboundid.ldap.sdk.LDAPException; +import com.unboundid.ldap.sdk.ResultCode; + +/** + * TODO javadoc + * + * @author lpouzenc + */ +public class LDAPConnectionWrapper implements Closeable { + + private final LDAPConnection conn; + + /** + * TODO javadoc + * @param host + * @param port + * @param bindDN + * @param password + */ + public LDAPConnectionWrapper(String host, int port, String bindDN, String password) { + LDAPConnectionOptions options = new LDAPConnectionOptions(); + options.setAbandonOnTimeout(true); + options.setAllowConcurrentSocketFactoryUse(true); + options.setAutoReconnect(true); + options.setCaptureConnectStackTrace(true); + options.setConnectTimeoutMillis(2000); // 2 seconds + options.setResponseTimeoutMillis(5000); // 5 seconds + options.setUseSynchronousMode(false); + + BindResult bindResult=null; + try { + conn = new LDAPConnection(options, host, port); + bindResult = conn.bind(bindDN, password); + } + catch (LDAPException e) { + throw new RuntimeException(e); + } + + ResultCode resultCode = bindResult.getResultCode(); + if ( resultCode != ResultCode.SUCCESS ) { + throw new RuntimeException("LDAP Bind failed : " + resultCode); + } + } + + /** + * Builds a new reader against current connection and a LDAP baseDN. + * + * @param dataSourceName Short name of this data source (for logging) + * @param baseDN Search base DN (will return childs of this DN) + * @param keyAttr Attribute name that is the primary key of the entry, identifying the entry in a unique manner + * @param lookAheadAmount Grab this amount of entries at once (in memory-sorted, 128 could be great) + * @return A new reader ready to iterate on search results + */ + public LDAPFlatDataReader newFlatReader(String dataSourceName, String baseDN, String keyAttr, int lookAheadAmount) { + try { + return new LDAPFlatDataReader(dataSourceName, conn, baseDN, keyAttr, lookAheadAmount); + } catch (LDAPException e) { + throw new RuntimeException(e); + } + } + + /** + * Builds a new writer that could insert/update/delete entries on a particular LDAP connection and baseDN. + * + * @param baseDN Search base DN (will return childs of this DN) + * @param keyAttr Attribute name that is the primary key of the entry, identifying the entry in a unique manner + * @return A new writter limited on a particular baseDN + */ + public LDAPFlatDataWriter newFlatWriter(String baseDN, String keyAttr) { + try { + return new LDAPFlatDataWriter(conn, baseDN, keyAttr); + } catch (LDAPException e) { + throw new RuntimeException(e); + } + } + + /** + * Close the current ldap connection. + */ + @Override + public void close() throws IOException { + this.conn.close(); + } +} diff --git a/src/connectors/src/data/io/ldap/LDAPFlatDataReader.java b/src/connectors/src/data/io/ldap/LDAPFlatDataReader.java new file mode 100644 index 0000000..2cc79a8 --- /dev/null +++ b/src/connectors/src/data/io/ldap/LDAPFlatDataReader.java @@ -0,0 +1,178 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.io.ldap; + +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +import com.unboundid.ldap.sdk.Attribute; +import com.unboundid.ldap.sdk.Filter; +import com.unboundid.ldap.sdk.LDAPConnection; +import com.unboundid.ldap.sdk.LDAPException; +import com.unboundid.ldap.sdk.SearchRequest; +import com.unboundid.ldap.sdk.SearchResult; +import com.unboundid.ldap.sdk.SearchResultEntry; +import com.unboundid.ldap.sdk.SearchResultListener; +import com.unboundid.ldap.sdk.SearchResultReference; +import com.unboundid.ldap.sdk.SearchScope; + +import data.MVDataEntry; +import data.io.AbstractMVDataReader; + +/** + * Stream-oriented reader from a particular LDAP connection + * Always returns lines/items sorted by lexicographical ascending key + * Consistent even if there is a Writer on same LDAP connection (useful for sync) + * + * @author lpouzenc + */ +public class LDAPFlatDataReader extends AbstractMVDataReader { + + private final LDAPConnection conn; + private final String baseDN; + private final String keyAttr; + private final int lookAheadAmount; + private final SortedSet<String> keys; + + private transient Iterator<String> keysItCached; + private transient Iterator<String> keysItConsumed; + private transient SortedMap<String, MVDataEntry> entries; + + // Listener to feed LDAP search result in SortedMap without instantiating a big fat SearchResult + private final SearchResultListener keysReqListener = new SearchResultListener() { + private static final long serialVersionUID = 3364745402521913458L; + + @Override + public void searchEntryReturned(SearchResultEntry searchEntry) { + keys.add(searchEntry.getAttributeValue(keyAttr)); + } + + @Override + public void searchReferenceReturned(SearchResultReference searchReference) { + throw new RuntimeException("Unsupported : search request for all '" + keyAttr + "' has returned at least one reference (excepected : an entry)"); + } + }; + + /** + * Construct a new reader that wrap a particular LDAP search on a given connection + * @param dataSourceName Short name of this data source (for logging) + * @param conn Already initialized LDAP connection where run the search + * @param baseDN Search base DN (will return childs of this DN) + * @param keyAttr Attribute name that is the primary key of the entry, identifying the entry in a unique manner + * @param lookAheadAmount Grab this amount of entries at once (in memory-sorted, 128 could be great) + * @throws LDAPException + */ + public LDAPFlatDataReader(String dataSourceName, LDAPConnection conn, String baseDN, String keyAttr, int lookAheadAmount) throws LDAPException { + this.dataSourceName = dataSourceName; + this.conn = conn; + this.baseDN = baseDN; + this.keyAttr = keyAttr; + this.lookAheadAmount = lookAheadAmount; + + // Grab all the entries' keys from LDAP connection and put them in this.keys + this.keys = new TreeSet<String>(); + SearchRequest keysReq = new SearchRequest(keysReqListener, baseDN, SearchScope.ONE, Filter.create("(objectClass=*)"), keyAttr); + conn.search(keysReq); + } + + /** + * {@inheritDoc} + * Note : multiple iterators on the same instance are not supported + */ + @Override + public Iterator<MVDataEntry> iterator() { + // Reset the search (it uses two different iterators on the same set) + keysItCached = keys.iterator(); + keysItConsumed = keys.iterator(); + entries = new TreeMap<String, MVDataEntry>(); + + return this; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean hasNext() { + return (keysItConsumed==null)?false:keysItConsumed.hasNext(); + } + + /** + * {@inheritDoc} + */ + @Override + public MVDataEntry next() { + String wantedKey = keysItConsumed.next(); + + // Feed the lookAhead buffer if it is empty (and there is more elements to grab) + if ( entries.isEmpty() && keysItCached.hasNext() ) { + lookAhead(lookAheadAmount); + } + + //FIXME : it is possible to have inconsistency between "entries" content and keysIt* values if some entry is deleted since we have read all the keys + + // Pop an entry from the lookAhead buffer + MVDataEntry wantedEntry = entries.remove(wantedKey); + if ( wantedEntry == null ) { + throw new NoSuchElementException(); + } + + return wantedEntry; + } + + /** + * Performs look-ahead of amount entries, using the next sorted keys previously queried. + * @param amount + */ + private void lookAhead(int amount) { + if ( amount < 1 ) { + throw new IllegalArgumentException("LookAhead amount has to be >= 1"); + } + try { + // Build a search that matches "amount" next entries + Filter filter = Filter.createEqualityFilter(keyAttr, keysItCached.next()); + for (int i=0; ( i < amount-1 ) && keysItCached.hasNext(); i++) { + filter = Filter.createORFilter(filter, Filter.createEqualityFilter(keyAttr, keysItCached.next())); + } + SearchRequest searchRequest = new SearchRequest(baseDN, SearchScope.ONE, filter, "*"); + + // XXX Could use a second listener, as for the keys + // Get all this entries in memory, convert them in MVDataEntry beans and store them in a SortedMap + SearchResult search = conn.search(searchRequest); + + for (SearchResultEntry ldapEntry: search.getSearchEntries()) { + String key = ldapEntry.getAttributeValue(keyAttr); + MVDataEntry mvEntry = new MVDataEntry(key); + + for ( Attribute attr : ldapEntry.getAttributes() ) { + mvEntry.put(attr.getName(), attr.getValues()); + } + entries.put(key, mvEntry); + } + } catch (LDAPException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/connectors/src/data/io/ldap/LDAPFlatDataWriter.java b/src/connectors/src/data/io/ldap/LDAPFlatDataWriter.java new file mode 100644 index 0000000..d1b8918 --- /dev/null +++ b/src/connectors/src/data/io/ldap/LDAPFlatDataWriter.java @@ -0,0 +1,198 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.io.ldap; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.unboundid.ldap.sdk.Attribute; +import com.unboundid.ldap.sdk.DN; +import com.unboundid.ldap.sdk.DeleteRequest; +import com.unboundid.ldap.sdk.Entry; +import com.unboundid.ldap.sdk.LDAPConnection; +import com.unboundid.ldap.sdk.LDAPException; +import com.unboundid.ldap.sdk.Modification; +import com.unboundid.ldap.sdk.ModificationType; +import com.unboundid.ldap.sdk.ModifyRequest; +import com.unboundid.ldap.sdk.RDN; +import com.unboundid.ldap.sdk.schema.EntryValidator; +import com.unboundid.ldif.LDIFException; + +import data.MVDataEntry; +import data.io.AbstractMVDataWriter; + +/** + * Stream-oriented LDAP writer from a particular LDAP Directory connection. + * + * @author lpouzenc + */ +public class LDAPFlatDataWriter extends AbstractMVDataWriter { + + private final LDAPConnection conn; + private final DN baseDN; + private final String keyAttr; + private final EntryValidator validator; + + /** + * Construct a new writer that could insert/update/delete entries on a particular LDAP connection and baseDN. + * + * @param conn Already initialized LDAP connection where run the search + * @param baseDN Search base DN (will return childs of this DN) + * @param keyAttr Attribute name that is the primary key of the entry, identifying the entry in a unique manner + * @throws LDAPException + */ + public LDAPFlatDataWriter(LDAPConnection conn, String baseDN, String keyAttr) throws LDAPException { + this.conn = conn; + this.baseDN = new DN(baseDN); + this.keyAttr = keyAttr; + this.validator = new EntryValidator(conn.getSchema()); + } + + /** + * {@inheritDoc} + */ + @Override + public void insert(MVDataEntry newEntry) throws LDAPException { + // Build the DN + DN dn = new DN(new RDN(keyAttr, newEntry.getKey()), baseDN); + + // Convert storage objects + Collection<Attribute> attributes = new ArrayList<Attribute>(); + for ( Map.Entry<String, String> entry : newEntry.getAllEntries() ) { + attributes.add(new Attribute(entry.getKey(), entry.getValue())); + } + Entry newLDAPEntry = new Entry(dn, attributes); + + // Add the entry + if ( dryRun ) { + // In dry-run mode, validate the entry + ArrayList<String> invalidReasons = new ArrayList<String>(5); + boolean valid = validator.entryIsValid(newLDAPEntry, invalidReasons); + if ( !valid ) throw new RuntimeException( + "Entry validator has failed to verify this entry :\n" + newLDAPEntry.toLDIFString() + + "Reasons are :\n" + invalidReasons); + } else { + // In real-run mode, insert the entry + try { + conn.add(newLDAPEntry); + } catch (LDAPException e) { + throw new LDAPException(e.getResultCode(), "Error while inserting this entry :\n" + newLDAPEntry.toLDIFString(), e); + } + } + } + + /** + * {@inheritDoc} + */ + @Override + public void update(MVDataEntry updatedEntry, MVDataEntry originalEntry, Set<String> attrToUpdate) throws LDAPException, LDIFException { + // Build the DN + DN dn = new DN(new RDN(keyAttr, updatedEntry.getKey()), baseDN); + + // Convert storage objects + List<Modification> mods = new ArrayList<Modification>(); + for ( String attr : attrToUpdate ) { + Set<String> originalValues = originalEntry.getValues(attr); + Set<String> updatedValues = updatedEntry.getValues(attr); + + Modification modification = null; + + if ( updatedValues.isEmpty() ) { + modification = new Modification(ModificationType.DELETE, attr); + } else { + String[] updatedValuesArr = updatedValues.toArray(new String[0]); + + if ( originalValues.isEmpty() ) { + modification = new Modification(ModificationType.ADD, attr, updatedValuesArr); + } else { + modification = new Modification(ModificationType.REPLACE, attr, updatedValuesArr); + } + } + + mods.add(modification); + } + ModifyRequest modReq = new ModifyRequest(dn, mods); + + // Update the entry + if ( dryRun ) { + // Simulate originalEntry update + Collection<Attribute> attributes = new ArrayList<Attribute>(); + for ( Map.Entry<String, String> entry : originalEntry.getAllEntries() ) { + attributes.add(new Attribute(entry.getKey(), entry.getValue())); + } + Entry originalLDAPEntry = new Entry(dn, attributes); + + // Warning : Unboundid SDK is okay with mandatory attributes with value "" (empty string) + // OpenLDAP do not allow that empty strings in mandatory attributes. + // Empty strings are discarded by MVDataEntry.put() for now. + Entry modifiedLDAPEntry; + try { + modifiedLDAPEntry = Entry.applyModifications(originalLDAPEntry, false, mods); + } catch (LDAPException originalException) { + throw new RuntimeException("Entry update simulation has failed while running applyModifications()\n" + + "original entry : " + originalEntry + "\n" + + "wanted updated entry : " + updatedEntry + "\n" + + "modification request : " + modReq, + originalException); + } + ArrayList<String> invalidReasons = new ArrayList<String>(5); + boolean valid = validator.entryIsValid(modifiedLDAPEntry, invalidReasons); + if ( !valid ) throw new RuntimeException("Entry update simulation has failed while checking entryIsValid()\n" + + "modified entry : " + modifiedLDAPEntry.toLDIFString() + "\n" + + "reasons :" + invalidReasons); + } else { + // In real-run mode, update the entry + try { + conn.modify(modReq); + } catch (LDAPException originalException) { + throw new LDAPException(originalException.getResultCode(), + "Error while updating this entry :\n" + modReq.toLDIFString(), + originalException); + } + } + } + + /** + * {@inheritDoc} + */ + @Override + public void delete(MVDataEntry existingEntry) throws LDAPException { + // Build the DN + DN dn = new DN(new RDN(keyAttr, existingEntry.getKey()), baseDN); + + // Delete the entry + try { + if ( dryRun ) { + //XXX : try to verify the entry existence in dry-run mode ? + } else { + conn.delete(new DeleteRequest(dn)); + } + } catch (LDAPException originalException) { + throw new LDAPException(originalException.getResultCode(), + "Error while deleting this dn : " + dn.toString(), + originalException); + } + } + +} diff --git a/src/connectors/src/data/io/sql/SQLConnectionWrapper.java b/src/connectors/src/data/io/sql/SQLConnectionWrapper.java new file mode 100644 index 0000000..2bab2c8 --- /dev/null +++ b/src/connectors/src/data/io/sql/SQLConnectionWrapper.java @@ -0,0 +1,136 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.io.sql; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.SQLException; + +import data.io.MVDataReader; + +/** + * TODO javadoc + * + * @author lpouzenc + */ +public class SQLConnectionWrapper implements Closeable { + + /** + * Enumeration of supported DBMS. Each use a particular JDBC driver. + */ + public enum DBMSType { oracle, mysql/*, derby*/ } + + private final Connection conn; + + /** + * TODO javadoc + * @param dbms + * @param host + * @param port + * @param ress + * @param user + * @param pass + * @param db + */ + public SQLConnectionWrapper(DBMSType dbms, String host, int port, String ress, String user, String pass, String db) { + + String driverClassName=null; + String url; + + switch ( dbms ) { + case oracle: + driverClassName="oracle.jdbc.driver.OracleDriver"; + url="jdbc:oracle:thin:@" + host + ":" + port + ":" + ress + "/" + db; + break; + case mysql: + driverClassName="com.mysql.jdbc.Driver"; + url="jdbc:mysql://" + host + ":" + port + "/" + db; + break; + /* Could be useful with JUnit tests + case derby: + driverClassName="org.apache.derby.jdbc.EmbeddedDriver"; + url="jdbc:derby:" + db; + break; + */ + default: + throw new IllegalArgumentException("Unsupported DBMSType : " + dbms); + } + + try { + @SuppressWarnings("unchecked") + Class<? extends Driver> clazz = (Class<? extends Driver>) Class.forName(driverClassName); + DriverManager.registerDriver(clazz.newInstance()); + } catch (Exception e) { + throw new RuntimeException("Can't load or register JDBC driver for " + dbms + " (" + driverClassName + ")", e); + } + + try { + conn = DriverManager.getConnection(url, user, pass); + } catch (SQLException e) { + throw new RuntimeException("Can't establish database connection (" + url + ")"); + } + } + + /** + * Builds a new reader against current connection and a File containing a SELECT statement. + * @param name + * @param queryFile + * @return + * @throws IOException + */ + public MVDataReader newReader(String name, File queryFile) throws IOException { + return new SQLRelDataReader(name, conn, queryFile); + } + + /** + * Builds a new reader against current connection and a String containing a SELECT statement. + * @param name + * @param query + * @return + * @throws IOException + */ + public MVDataReader newReader(String name, String query) { + return new SQLRelDataReader(name, conn, query); + } + + /** + * Close the current database connection. + */ + @Override + public void close() throws IOException { + try { + conn.close(); + } catch (SQLException e) { + throw new IOException("Exception occured while trying to close the SQL connection", e); + } + } + + /** + * @return the current database connection (useful for JUnit tests) + */ + public Connection getConn() { + return conn; + } +} diff --git a/src/connectors/src/data/io/sql/SQLRelDataReader.java b/src/connectors/src/data/io/sql/SQLRelDataReader.java new file mode 100644 index 0000000..b6355e9 --- /dev/null +++ b/src/connectors/src/data/io/sql/SQLRelDataReader.java @@ -0,0 +1,173 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr> + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see <http://www.gnu.org/licenses/> + */ + +package data.io.sql; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Iterator; + +import data.MVDataEntry; +import data.io.AbstractMVDataReader; + +/** + * Stream-oriented reader from a particular RDBMS source. + * + * @author lpouzenc + */ +public class SQLRelDataReader extends AbstractMVDataReader { + + private final Connection conn; + private final String request; + + private transient String columnNames[]; + private transient ResultSet rs; + private transient boolean didNext; + private transient boolean hasNext; + + /** + * Build a new reader from an existing connection and a File containing a SELECT statement. + * @param dataSourceName A short string representing this reader (for logging) + * @param conn A pre-established SQL data connection + * @param queryFile An SQL file containing an SQL SELECT statement + * @throws IOException + */ + public SQLRelDataReader(String dataSourceName, Connection conn, File queryFile) throws IOException { + this.dataSourceName = dataSourceName; + this.conn = conn; + this.request = readEntireFile(queryFile); + } + + /** + * Build a new reader from an existing connection and a String containing a SELECT statement. + * @param dataSourceName A short string representing this reader (for logging) + * @param conn A pre-established SQL data connection + * @param query A String containing an SQL SELECT statement + * @throws IOException + */ + public SQLRelDataReader(String dataSourceName, Connection conn, String query) { + this.dataSourceName = dataSourceName; + this.conn = conn; + this.request = query; + } + + /** + * {@inheritDoc} + * Note : multiple iterators on the same instance are not supported + */ + @Override + public Iterator<MVDataEntry> iterator() { + try { + // Reset iterator-related attributes + hasNext = false; + didNext = false; + + // Close and free any previous request result + if ( rs != null ) { + rs.close(); + } + // (Re-)Execute the SQL request + Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + rs = stmt.executeQuery(request); + + // Get the column names + ResultSetMetaData rsmd = rs.getMetaData(); + columnNames = new String[rsmd.getColumnCount()]; + for (int i = 0; i < columnNames.length ; i++) { + // Java SQL : all indices starts at 1 (it sucks !) + columnNames[i] = rsmd.getColumnName(i+1); + } + } catch (SQLException e) { + throw new RuntimeException("Could not execute query : " + e.getMessage() + "\n" + request ); + } + + return this; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean hasNext() { + // java.sql.ResultSet don't implement Iterable interface at all + // It's next() don't return anything except hasNext() result but it moves the cursor ! + if (!didNext) { + try { + hasNext = rs.next(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + didNext = true; + } + return hasNext; + } + + /** + * {@inheritDoc} + */ + @Override + public MVDataEntry next() { + MVDataEntry result = null; + try { + if (!didNext) { + rs.next(); + } + didNext = false; + //TODO Instead of always use the first col, user could choose a specific columnName like in LDAP + String key = rs.getString(1); + result = new MVDataEntry(key); + for (int i = 0; i < columnNames.length ; i++) { + // Java SQL : all indices starts at 1 (it sucks !) + result.splitAndPut(columnNames[i], rs.getString(i+1), ";"); // TODO regex should be an option + } + + } catch (SQLException e) { + throw new RuntimeException("Exception while reading next line in SQL resultset", e); + } + + return result; + } + + /** + * Helper function to load and entire file as a String. + * @param file + * @return + * @throws IOException + */ + private static String readEntireFile(File file) throws IOException { + FileReader input = new FileReader(file); + StringBuilder contents = new StringBuilder(); + char[] buffer = new char[4096]; + int read = 0; + do { + contents.append(buffer, 0, read); + read = input.read(buffer); + } while (read >= 0); + input.close(); + + return contents.toString(); + } +} |