001 package de.deepamehta.plugins.csv;
002
003 import java.io.File;
004 import java.io.FileNotFoundException;
005 import java.io.FileReader;
006 import java.io.IOException;
007 import java.util.ArrayList;
008 import java.util.Arrays;
009 import java.util.HashMap;
010 import java.util.List;
011 import java.util.Map;
012 import java.util.logging.Logger;
013
014 import javax.ws.rs.HeaderParam;
015 import javax.ws.rs.POST;
016 import javax.ws.rs.Path;
017 import javax.ws.rs.PathParam;
018 import javax.ws.rs.WebApplicationException;
019
020 import au.com.bytecode.opencsv.CSVReader;
021 import de.deepamehta.core.AssociationDefinition;
022 import de.deepamehta.core.RelatedTopic;
023 import de.deepamehta.core.TopicType;
024 import de.deepamehta.core.model.CompositeValueModel;
025 import de.deepamehta.core.model.TopicModel;
026 import de.deepamehta.core.osgi.PluginActivator;
027 import de.deepamehta.core.service.ClientState;
028 import de.deepamehta.core.service.PluginService;
029 import de.deepamehta.core.service.annotation.ConsumesService;
030 import de.deepamehta.core.storage.spi.DeepaMehtaTransaction;
031 import de.deepamehta.plugins.files.ResourceInfo;
032 import de.deepamehta.plugins.files.service.FilesService;
033
034 @Path("csv")
035 public class CsvPlugin extends PluginActivator {
036
037 private static Logger log = Logger.getLogger(CsvPlugin.class.getName());
038
039 public static final String FOLDER = "csv";
040
041 public static final char SEPARATOR = '|';
042
043 private boolean isInitialized;
044
045 private FilesService fileService;
046
047 @POST
048 @Path("import/{uri}/{file}")
049 public ImportStatus importCsv(//
050 @PathParam("uri") String typeUri,//
051 @PathParam("file") long fileId,//
052 @HeaderParam("Cookie") ClientState cookie) {
053
054 DeepaMehtaTransaction tx = dms.beginTx();
055 try {
056 List<String[]> lines = readCsvFile(fileId);
057 if (lines.size() < 2) {
058 return new ImportStatus(false, "please upload a valid CSV, see README", null);
059 }
060
061 List<String> childTypeUris = Arrays.asList(lines.get(0));
062 String uriPrefix = childTypeUris.get(0);
063
064 Map<String, Long> topicsByUri = getTopicIdsByUri(typeUri);
065 Map<String, Map<String, Long>> aggrIdsByTypeUriAndValue = getPossibleAggrChilds(typeUri, childTypeUris);
066
067 // status informations
068 int created = 0, deleted = 0, updated = 0;
069
070 // persist each row
071 for (int r = 1; r < lines.size(); r++) {
072 String[] row = lines.get(r);
073 String topicUri = uriPrefix + "." + row[0];
074
075 // create a fresh model
076 TopicModel model = new TopicModel(typeUri);
077 model.setUri(topicUri);
078
079 // map all columns to composite value
080 CompositeValueModel value = new CompositeValueModel();
081 for (int c = 1; c < row.length; c++) {
082 String childTypeUri = childTypeUris.get(c);
083 String childValue = row[c];
084
085 // reference or create a child
086 Map<String, Long> aggrIdsByValue = aggrIdsByTypeUriAndValue.get(childTypeUri);
087 if (aggrIdsByValue != null && aggrIdsByValue.get(childValue) != null) {
088 value.putRef(childTypeUri, aggrIdsByValue.get(childValue));
089 } else {
090 value.put(childTypeUri, childValue);
091 }
092 }
093 model.setCompositeValue(value);
094
095 // create or update a topic
096 Long topicId = topicsByUri.get(topicUri);
097 if (topicId == null) { // create
098 dms.createTopic(model, cookie);
099 created++;
100 } else { // update topic and remove from map
101 model.setId(topicId);
102 topicsByUri.remove(topicUri);
103 dms.updateTopic(model, cookie);
104 updated++;
105 }
106 }
107
108 // delete the remaining instances
109 for (String topicUri : topicsByUri.keySet()) {
110 Long topicId = topicsByUri.get(topicUri);
111 dms.deleteTopic(topicId);
112 deleted++;
113 }
114
115 List<String> status = new ArrayList<String>();
116 status.add("created: " + created);
117 status.add("updated: " + updated);
118 status.add("deleted: " + deleted);
119
120 tx.success();
121 return new ImportStatus(true, "SUCCESS", status);
122 } catch (IOException e) {
123 throw new RuntimeException(e) ;
124 } finally {
125 tx.finish();
126 }
127 }
128
129 /**
130 * get all possible aggregation instances and hash them by typeUri and value
131 *
132 * @param typeUri
133 * @param childTypeUris
134 * @return
135 */
136 private Map<String, Map<String, Long>> getPossibleAggrChilds(String typeUri, List<String> childTypeUris) {
137 TopicType topicType = dms.getTopicType(typeUri);
138 Map<String, Map<String, Long>> aggrIdsByTypeUriAndValue = new HashMap<String, Map<String, Long>>();
139 for (AssociationDefinition associationDefinition : topicType.getAssocDefs()) {
140 if (associationDefinition.getTypeUri().equals("dm4.core.aggregation_def")) {
141 String childTypeUri = associationDefinition.getChildTypeUri();
142 if (childTypeUris.contains(childTypeUri)) {
143 aggrIdsByTypeUriAndValue.put(childTypeUri, getTopicIdsByValue(childTypeUri));
144 }
145 }
146 }
147 return aggrIdsByTypeUriAndValue;
148 }
149
150 /**
151 * get all existing instance topics and hash them by value
152 *
153 * @param childTypeUri
154 * @return instance topics hashed by value
155 */
156 private Map<String, Long> getTopicIdsByValue(String childTypeUri) {
157 Map<String, Long> idsByValue = new HashMap<String, Long>();
158 for (RelatedTopic instance : dms.getTopics(childTypeUri, false, 0).getItems()) {
159 idsByValue.put(instance.getSimpleValue().toString(), instance.getId());
160 }
161 return idsByValue;
162 }
163
164 /**
165 * get all existing instance topics and hash them by URI
166 *
167 * @param typeUri
168 * @return instance topics hashed by URI
169 */
170 private Map<String, Long> getTopicIdsByUri(String typeUri) {
171 Map<String, Long> idsByUri = new HashMap<String, Long>();
172 for (RelatedTopic topic : dms.getTopics(typeUri, false, 0).getItems()) {
173 String topicUri = topic.getUri();
174 if (topicUri != null && topicUri.isEmpty() == false) {
175 idsByUri.put(topicUri, topic.getId());
176 }
177 }
178 return idsByUri;
179 }
180
181 /**
182 * read and validate CSV file
183 *
184 * @param fileId
185 * @return parsed CSV rows with trimmed column array
186 *
187 * @throws FileNotFoundException
188 * @throws IOException
189 */
190 private List<String[]> readCsvFile(long fileId) throws IOException {
191 String fileName = fileService.getFile(fileId).getAbsolutePath();
192 log.info("read CSV " + fileName);
193 CSVReader csvReader = new CSVReader(new FileReader(fileName), SEPARATOR);
194 List<String[]> lines = csvReader.readAll();
195 csvReader.close();
196
197 // trim all columns
198 for (String[] row : lines) {
199 for (int col = 0; col < row.length; col++) {
200 row[col] = row[col].trim();
201 }
202 }
203 return lines;
204 }
205
206 /**
207 * Initialize.
208 */
209 @Override
210 public void init() {
211 isInitialized = true;
212 configureIfReady();
213 }
214
215 @Override
216 @ConsumesService("de.deepamehta.plugins.files.service.FilesService")
217 public void serviceArrived(PluginService service) {
218 if (service instanceof FilesService) {
219 fileService = (FilesService) service;
220 }
221 configureIfReady();
222 }
223
224 @Override
225 public void serviceGone(PluginService service) {
226 if (service == fileService) {
227 fileService = null;
228 }
229 }
230
231 private void configureIfReady() {
232 if (isInitialized && fileService != null) {
233 createCsvDirectory();
234 }
235 }
236
237 private void createCsvDirectory() {
238 // TODO move the initialization to migration "0"
239 try {
240 ResourceInfo resourceInfo = fileService.getResourceInfo(FOLDER);
241 String kind = resourceInfo.toJSON().getString("kind");
242 if (kind.equals("directory") == false) {
243 String repoPath = System.getProperty("dm4.filerepo.path");
244 throw new IllegalStateException("CSV storage directory " + //
245 repoPath + File.separator + FOLDER + " can not be used");
246 }
247 } catch (WebApplicationException e) { // !exists
248 // catch fileService info request error => create directory
249 if (e.getResponse().getStatus() != 404) {
250 throw e;
251 } else {
252 log.info("create CSV directory");
253 fileService.createFolder(FOLDER, "/");
254 }
255 } catch (Exception e) {
256 throw new RuntimeException(e);
257 }
258 }
259 }