From 00f00b1055fa95d10360d1423d9c5f74c438fd2e Mon Sep 17 00:00:00 2001 From: Ludovic Pouzenc Date: Thu, 23 Oct 2014 12:26:30 +0200 Subject: Config : separate Source and Dest beans, move factory out of there --- src/main/src/conf/ConfigDestBean.java | 81 +++++++++++ src/main/src/conf/ConfigGlobalsBean.java | 2 +- src/main/src/conf/ConfigSourceBean.java | 105 +++++++++++++++ src/main/src/conf/ConfigSrcOrDestBean.java | 96 ------------- src/main/src/conf/ConfigTaskBean.java | 16 +-- src/main/src/conf/SSSyncConnectionsFactory.java | 61 --------- src/main/src/conf/SSSyncTasksFactory.java | 147 -------------------- src/main/src/sync/SSSyncTasksFactory.java | 170 ++++++++++++++++++++++++ 8 files changed, 365 insertions(+), 313 deletions(-) create mode 100644 src/main/src/conf/ConfigDestBean.java create mode 100644 src/main/src/conf/ConfigSourceBean.java delete mode 100644 src/main/src/conf/ConfigSrcOrDestBean.java delete mode 100644 src/main/src/conf/SSSyncConnectionsFactory.java delete mode 100644 src/main/src/conf/SSSyncTasksFactory.java create mode 100644 src/main/src/sync/SSSyncTasksFactory.java diff --git a/src/main/src/conf/ConfigDestBean.java b/src/main/src/conf/ConfigDestBean.java new file mode 100644 index 0000000..2d9b2f7 --- /dev/null +++ b/src/main/src/conf/ConfigDestBean.java @@ -0,0 +1,81 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see + */ + +package conf; + +/** + * Generated Configuration Bean + */ +public class ConfigDestBean { + public enum DestinationKind { ldap }; + + private String name; + private DestinationKind kind; + private String conn; + private String attr; + private String base; + private int lookahead = 128; /* Only for ldap kind, query lookahead entries at once */ + + public String getName() { + return name; + } + public void setName(String name) { + this.name = name; + } + public DestinationKind getKind() { + return kind; + } + public void setKind(DestinationKind kind) { + this.kind = kind; + } + public String getConn() { + return conn; + } + public void setConn(String conn) { + this.conn = conn; + } + public String getAttr() { + return attr; + } + public void setAttr(String attr) { + this.attr = attr; + } + public String getBase() { + return base; + } + public void setBase(String base) { + this.base = base; + } + public int getLookahead() { + return lookahead; + } + public void setLookahead(int lookahead) { + this.lookahead = lookahead; + } + + @Override + public String toString() { + return "ConfigDestBean [name=" + name + ", kind=" + kind + ", conn=" + + conn + ", attr=" + attr + ", base=" + base + ", lookahead=" + + lookahead + "]"; + } + + +} diff --git a/src/main/src/conf/ConfigGlobalsBean.java b/src/main/src/conf/ConfigGlobalsBean.java index 256acee..20243f3 100644 --- a/src/main/src/conf/ConfigGlobalsBean.java +++ b/src/main/src/conf/ConfigGlobalsBean.java @@ -24,7 +24,7 @@ package conf; * Generated Configuration Bean */ public class ConfigGlobalsBean { - private int maxExecTime; + private int maxExecTime = 5; public int getMaxExecTime() { return maxExecTime; diff --git a/src/main/src/conf/ConfigSourceBean.java b/src/main/src/conf/ConfigSourceBean.java new file mode 100644 index 0000000..cf01dce --- /dev/null +++ b/src/main/src/conf/ConfigSourceBean.java @@ -0,0 +1,105 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see + */ + +package conf; + +import data.filters.MVDataCombiner; + +/** + * Generated Configuration Bean + */ +public class ConfigSourceBean { + + //TODO : fixed_csv ? + public enum SourceKind { csv, ldap, sorted_csv, sql }; + + private String name; + private SourceKind kind; + private String conn; + private MVDataCombiner.MVDataCombineMode mode; + private String query; + private String path; + private String attr; + private String base; + private int lookahead = 128; /* Only for ldap kind, query lookahead entries at once */ + + public String getName() { + return name; + } + public void setName(String name) { + this.name = name; + } + public SourceKind getKind() { + return kind; + } + public void setKind(SourceKind kind) { + this.kind = kind; + } + public String getConn() { + return conn; + } + public void setConn(String conn) { + this.conn = conn; + } + public MVDataCombiner.MVDataCombineMode getMode() { + return mode; + } + public void setMode(MVDataCombiner.MVDataCombineMode mode) { + this.mode = mode; + } + public String getQuery() { + return query; + } + public void setQuery(String query) { + this.query = query; + } + public String getPath() { + return path; + } + public void setPath(String path) { + this.path = path; + } + public String getAttr() { + return attr; + } + public void setAttr(String attr) { + this.attr = attr; + } + public String getBase() { + return base; + } + public void setBase(String base) { + this.base = base; + } + public int getLookahead() { + return lookahead; + } + public void setLookahead(int lookahead) { + this.lookahead = lookahead; + } + + @Override + public String toString() { + return "ConfigSrcOrDestBean [name=" + name + ", kind=" + kind + + ", conn=" + conn + ", mode=" + mode + ", query=" + query + + ", path=" + path + ", attr=" + attr + ", base=" + base + ", lookahead=" + + lookahead + "]"; + } +} diff --git a/src/main/src/conf/ConfigSrcOrDestBean.java b/src/main/src/conf/ConfigSrcOrDestBean.java deleted file mode 100644 index 5be1674..0000000 --- a/src/main/src/conf/ConfigSrcOrDestBean.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes - * Copyright (C) 2014 Ludovic Pouzenc - * - * This file is part of SSSync. - * - * SSSync is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * SSSync is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with SSSync. If not, see - */ - -package conf; - -import data.filters.MVDataCombiner; - -/** - * Generated Configuration Bean - */ -public class ConfigSrcOrDestBean { - - public enum SourceKind { csv, ldap, sorted_csv, sql }; - - private String name; - private SourceKind kind; - private String conn; - private MVDataCombiner.MVDataCombineMode mode; - private String query; - private String path; - private String attr; - private String base; - - public String getName() { - return name; - } - public void setName(String name) { - this.name = name; - } - public SourceKind getKind() { - return kind; - } - public void setKind(SourceKind kind) { - this.kind = kind; - } - public String getConn() { - return conn; - } - public void setConn(String conn) { - this.conn = conn; - } - public MVDataCombiner.MVDataCombineMode getMode() { - return mode; - } - public void setMode(MVDataCombiner.MVDataCombineMode mode) { - this.mode = mode; - } - public String getQuery() { - return query; - } - public void setQuery(String query) { - this.query = query; - } - public String getPath() { - return path; - } - public void setPath(String path) { - this.path = path; - } - public String getAttr() { - return attr; - } - public void setAttr(String attr) { - this.attr = attr; - } - public String getBase() { - return base; - } - public void setBase(String base) { - this.base = base; - } - - @Override - public String toString() { - return "ConfigSrcOrDestBean [name=" + name + ", kind=" + kind - + ", conn=" + conn + ", mode=" + mode + ", query=" + query - + ", path=" + path + ", attr=" + attr + ", base=" + base + "]"; - } -} diff --git a/src/main/src/conf/ConfigTaskBean.java b/src/main/src/conf/ConfigTaskBean.java index ed34eee..11089d6 100644 --- a/src/main/src/conf/ConfigTaskBean.java +++ b/src/main/src/conf/ConfigTaskBean.java @@ -29,10 +29,10 @@ public class ConfigTaskBean { private String name; private ConfigOpLimitsBean opLimits; - private List sources; - private ConfigSrcOrDestBean destination; - private boolean skipReadErrors; - private boolean skipEntryDelete; + private List sources; + private ConfigDestBean destination; + private boolean skipReadErrors = false; + private boolean skipEntryDelete = false; public String getName() { return name; @@ -46,16 +46,16 @@ public class ConfigTaskBean { public void setOpLimits(ConfigOpLimitsBean opLimits) { this.opLimits = opLimits; } - public List getSources() { + public List getSources() { return sources; } - public void setSources(List sources) { + public void setSources(List sources) { this.sources = sources; } - public ConfigSrcOrDestBean getDestination() { + public ConfigDestBean getDestination() { return destination; } - public void setDestination(ConfigSrcOrDestBean destination) { + public void setDestination(ConfigDestBean destination) { this.destination = destination; } public boolean isSkipReadErrors() { diff --git a/src/main/src/conf/SSSyncConnectionsFactory.java b/src/main/src/conf/SSSyncConnectionsFactory.java deleted file mode 100644 index e747258..0000000 --- a/src/main/src/conf/SSSyncConnectionsFactory.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes - * Copyright (C) 2014 Ludovic Pouzenc - * - * This file is part of SSSync. - * - * SSSync is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * SSSync is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with SSSync. If not, see - */ - -package conf; - -import data.io.ConnectionsHolder; -import data.io.ldap.LDAPConnectionWrapper; -import data.io.sql.SQLConnectionWrapper; - -/** - * TODO javadoc - * - * @author lpouzenc - */ -public class SSSyncConnectionsFactory { - - /** - * Setup all connections described in config - * @return - * @throws Exception - */ - public static ConnectionsHolder setupConnections(ConfigConnectionsBean confConn) throws Exception { - ConnectionsHolder connections = new ConnectionsHolder(); - - for ( ConfigConnectionBean conn : confConn.getConnections() ) { - switch (conn.getType()) { - case jdbc: - SQLConnectionWrapper connSQL = new SQLConnectionWrapper(conn.getDbms(), conn.getHost(), conn.getPort(), conn.getRess(), conn.getUser(), conn.getPass(), conn.getDb()); - connections.putConnSQL(conn.getId(), connSQL); - break; - case ldap: - LDAPConnectionWrapper connLDAP = new LDAPConnectionWrapper(conn.getHost(), conn.getPort(), conn.getBind(), conn.getPass()); - connections.putConnLDAP(conn.getId(), connLDAP); - break; - default: - //XXX : find better Exception type - throw new Exception("Bad config : conn '" + conn.getId() + "' unsupported type"); - } - } - - return connections; - } - -} diff --git a/src/main/src/conf/SSSyncTasksFactory.java b/src/main/src/conf/SSSyncTasksFactory.java deleted file mode 100644 index de3e8f6..0000000 --- a/src/main/src/conf/SSSyncTasksFactory.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes - * Copyright (C) 2014 Ludovic Pouzenc - * - * This file is part of SSSync. - * - * SSSync is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * SSSync is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with SSSync. If not, see - */ - -package conf; - -import java.io.File; -import java.io.FileReader; -import java.util.ArrayList; -import java.util.List; - -import sync.BasicSyncTask; -import data.filters.MVDataCombiner; -import data.filters.MVDataCombiner.MVDataCombineMode; -import data.io.ConnectionsHolder; -import data.io.MVDataReader; -import data.io.MVDataWriter; -import data.io.SafeDataReader; -import data.io.csv.CSVDataReader; -import data.io.ldap.LDAPConnectionWrapper; -import data.io.sql.SQLConnectionWrapper; - -/** - * TODO javadoc - * - * @author lpouzenc - */ -public class SSSyncTasksFactory { - - /** - * Build tasks objects with all needed resources from a config beans tree - * @param conf - * @return - * @throws Exception - */ - public static List setupTasks(ConnectionsHolder connections, ConfigRootBean confMain) throws Exception { - List tasks = new ArrayList(); - - // For each task... - for ( ConfigTaskBean confTask: confMain.getTasks() ) { - MVDataReader srcReader=null; - - // Building all sources - - List confSources = confTask.getSources(); - // See if we are in multiple source situation (then MVDataCombiner) or not (then simple MVDataReader) - if ( confSources.size() == 0 ) { - throw new Exception("Bad config : task '" + confTask.getName() + "' has no defined sources"); - } else if ( confSources.size() == 1 ) { - srcReader = new SafeDataReader(_makeReader(connections, confSources.get(0), confTask.getName()), confTask.isSkipReadErrors()); - } else { - List readers = new ArrayList(); - List mergeModes = new ArrayList(); - - // For each source of the future MVDataCombiner... - for ( ConfigSrcOrDestBean confSource: confSources ) { - // Create and add the reader and his parameters - readers.add(new SafeDataReader(_makeReader(connections, confSource, confTask.getName()), confTask.isSkipReadErrors())); - mergeModes.add(confSource.getMode()); - } - - srcReader = new MVDataCombiner("srcCombiner", readers.toArray(new MVDataReader[0]), mergeModes.toArray(new MVDataCombineMode[0])); - } - - // Building destination - - MVDataReader dstReader=null; - MVDataWriter dstWriter=null; - - ConfigSrcOrDestBean confDestination = confTask.getDestination(); - switch ( confDestination.getKind() ) { - case ldap: - LDAPConnectionWrapper builder = connections.getLDAPConnectionBuilder(confDestination.getConn()); - // TODO : configurable lookAhead - MVDataReader tmpReader = builder.newFlatReader(confDestination.getName()+"_reader", confDestination.getBase(), confDestination.getAttr(), 128); - dstReader = new SafeDataReader(tmpReader, false); - dstWriter = builder.newFlatWriter(confDestination.getBase(), confDestination.getAttr()); - break; - default: - throw new Exception("Bad config : task '" + confTask.getName() + "' unsupported destination kind"); - } - - // Then building the sync task and add it to the task list - int maxInserts = confTask.getOpLimits().getInsert(); - int maxUpdates = confTask.getOpLimits().getUpdate(); - int maxDeletes = confTask.getOpLimits().getDelete(); - - BasicSyncTask task = new BasicSyncTask(confTask.getName(), false, srcReader, dstReader, dstWriter); - task.setOperationLimits(maxInserts, maxUpdates, maxDeletes); - task.setSkipEntryDelete(confTask.isSkipEntryDelete()); - tasks.add(task); - } - - return tasks; - } - - /** - * Helper function to make a new reader from an existing connection - * @param confSource - * @param taskName - * @return - * @throws Exception - */ - private static MVDataReader _makeReader(ConnectionsHolder connections, ConfigSrcOrDestBean confSource, String taskName) throws Exception { - MVDataReader reader=null; - switch (confSource.getKind()) { - case csv: - reader = new CSVDataReader(confSource.getName(), new FileReader(confSource.getPath()), false); - break; - case ldap: - LDAPConnectionWrapper ldapConnBuilder = connections.getLDAPConnectionBuilder(confSource.getConn()); - //FIXME : if conf error, get...ConnectionBuilder could return null - //TODO : configurable lookAhead - reader = ldapConnBuilder.newFlatReader(confSource.getName(), confSource.getBase(), confSource.getAttr(), 128); - break; - case sorted_csv: - reader = new CSVDataReader(confSource.getName(), new FileReader(confSource.getPath()), true); - break; - case sql: - SQLConnectionWrapper sqlConnBuilder = connections.getSQLConnectionBuilder(confSource.getConn()); - //TODO We assume the query config item is a filepath. It isn't checked anywhere. - reader = sqlConnBuilder.newReader(confSource.getName(), new File(confSource.getQuery())); - break; - default: - throw new Exception("Bad config : task '" + taskName + "' unsupported source kind"); - } - - return reader; - } - -} diff --git a/src/main/src/sync/SSSyncTasksFactory.java b/src/main/src/sync/SSSyncTasksFactory.java new file mode 100644 index 0000000..a91b3fa --- /dev/null +++ b/src/main/src/sync/SSSyncTasksFactory.java @@ -0,0 +1,170 @@ +/* + * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes + * Copyright (C) 2014 Ludovic Pouzenc + * + * This file is part of SSSync. + * + * SSSync is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SSSync is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SSSync. If not, see + */ + +package sync; + +import java.io.File; +import java.io.FileReader; +import java.util.ArrayList; +import java.util.List; + +import conf.ConfigDestBean; +import conf.ConfigRootBean; +import conf.ConfigSourceBean; +import conf.ConfigTaskBean; + +import sync.AbstractSyncTask; +import data.filters.MVDataCombiner; +import data.filters.MVDataCombiner.MVDataCombineMode; +import data.io.ConnectionsHolder; +import data.io.MVDataReadWriterPair; +import data.io.MVDataReader; +import data.io.MVDataWriter; +import data.io.SafeDataReader; +import data.io.csv.CSVDataReader; +import data.io.ldap.LDAPConnectionWrapper; +import data.io.sql.SQLConnectionWrapper; + +/** + * TODO javadoc + * + * @author lpouzenc + */ +public class SSSyncTasksFactory { + + + + /** + * Build tasks objects with all needed resources from a config bean tree + * + * @param conf + * @return + * @throws Exception + */ + public static List setupTasks(ConnectionsHolder connections, ConfigRootBean confMain) throws Exception { + List tasks = new ArrayList(); + + // For each task... + for ( ConfigTaskBean confTask: confMain.getTasks() ) { + MVDataReader srcReader=null; + + // Building all sources + + List confSources = confTask.getSources(); + // See if we are in multiple source situation (then MVDataCombiner) or not (then simple MVDataReader) + if ( confSources.size() == 0 ) { + throw new Exception("Bad config : task '" + confTask.getName() + "' has no defined sources"); + } else if ( confSources.size() == 1 ) { + srcReader = new SafeDataReader(_makeSrcReader(connections, confSources.get(0), confTask.getName()), confTask.isSkipReadErrors()); + } else { + List readers = new ArrayList(); + List mergeModes = new ArrayList(); + + // For each source of the future MVDataCombiner... + for ( ConfigSourceBean confSource: confSources ) { + // Create and add the reader and his parameters + readers.add(new SafeDataReader(_makeSrcReader(connections, confSource, confTask.getName()), confTask.isSkipReadErrors())); + mergeModes.add(confSource.getMode()); + } + + srcReader = new MVDataCombiner("srcCombiner", readers.toArray(new MVDataReader[0]), mergeModes.toArray(new MVDataCombineMode[0])); + } + + // Building destination + MVDataReadWriterPair dstRWriter = _makeDestRWriter(connections, confTask.getDestination(), confTask.getName()); + MVDataWriter dstWriter = dstRWriter.writer; + MVDataReader dstReader = new SafeDataReader(dstRWriter.reader, false); + + // Extract config opLimits + int maxInserts = confTask.getOpLimits().getInsert(); + int maxUpdates = confTask.getOpLimits().getUpdate(); + int maxDeletes = confTask.getOpLimits().getDelete(); + + // Building the sync task and add it to the task list + BasicSyncTask task = new BasicSyncTask(confTask.getName(), false, srcReader, dstReader, dstWriter); + task.setOperationLimits(maxInserts, maxUpdates, maxDeletes); + task.setSkipEntryDelete(confTask.isSkipEntryDelete()); + tasks.add(task); + } + + return tasks; + } + + /** + * Helper function to make a new reader from an existing connection + * @param connections + * @param confSource + * @param taskName + * @return + * @throws Exception + */ + private static MVDataReader _makeSrcReader(ConnectionsHolder connections, ConfigSourceBean confSource, String taskName) throws Exception { + MVDataReader reader=null; + switch (confSource.getKind()) { + case csv: + reader = new CSVDataReader(confSource.getName(), new FileReader(confSource.getPath()), false); + break; + case ldap: + LDAPConnectionWrapper ldapConnBuilder = connections.getLDAPConnectionBuilder(confSource.getConn()); + //FIXME if conf error, getLDAPConnectionBuilder could return null + reader = ldapConnBuilder.newFlatReader(confSource.getName(), confSource.getBase(), confSource.getAttr(), confSource.getLookahead()); + break; + case sorted_csv: + reader = new CSVDataReader(confSource.getName(), new FileReader(confSource.getPath()), true); + break; + case sql: + SQLConnectionWrapper sqlConnBuilder = connections.getSQLConnectionBuilder(confSource.getConn()); + //FIXME if conf error, getSQLConnectionBuilder could return null + //TODO We assume the query config item is a filepath. It isn't checked anywhere. + reader = sqlConnBuilder.newReader(confSource.getName(), new File(confSource.getQuery())); + break; + default: + throw new Exception("Bad config : task '" + taskName + "' unsupported source kind"); + } + + return reader; + } + + /** + * Helper function to make a new read-writer from an existing connection + * + * @param connections + * @param confDest + * @param taskName + * @return + * @throws Exception + */ + private static MVDataReadWriterPair _makeDestRWriter(ConnectionsHolder connections, ConfigDestBean confDest, String taskName) throws Exception { + MVDataReader reader = null; + MVDataWriter writer = null; + + switch ( confDest.getKind() ) { + case ldap: + LDAPConnectionWrapper builder = connections.getLDAPConnectionBuilder(confDest.getConn()); + reader = builder.newFlatReader(confDest.getName()+"_reader", confDest.getBase(), confDest.getAttr(), confDest.getLookahead()); + writer = builder.newFlatWriter(confDest.getBase(), confDest.getAttr()); + break; + default: + throw new Exception("Bad config : task '" + taskName + "' unsupported destination kind"); + } + + return new MVDataReadWriterPair(reader, writer); + } +} -- cgit v1.2.3