001 package de.deepamehta.plugins.csv;
002
003 import java.io.File;
004 import java.io.FileNotFoundException;
005 import java.io.FileReader;
006 import java.io.IOException;
007 import java.util.ArrayList;
008 import java.util.Arrays;
009 import java.util.HashMap;
010 import java.util.List;
011 import java.util.Map;
012 import java.util.logging.Logger;
013
014 import javax.ws.rs.HeaderParam;
015 import javax.ws.rs.POST;
016 import javax.ws.rs.Path;
017 import javax.ws.rs.PathParam;
018 import javax.ws.rs.WebApplicationException;
019
020 import au.com.bytecode.opencsv.CSVReader;
021 import de.deepamehta.core.AssociationDefinition;
022 import de.deepamehta.core.RelatedTopic;
023 import de.deepamehta.core.TopicType;
024 import de.deepamehta.core.model.CompositeValueModel;
025 import de.deepamehta.core.model.TopicModel;
026 import de.deepamehta.core.osgi.PluginActivator;
027 import de.deepamehta.core.service.ClientState;
028 import de.deepamehta.core.service.PluginService;
029 import de.deepamehta.core.service.annotation.ConsumesService;
030 import de.deepamehta.core.storage.spi.DeepaMehtaTransaction;
031 import de.deepamehta.plugins.files.ResourceInfo;
032 import de.deepamehta.plugins.files.service.FilesService;
033 import javax.ws.rs.Produces;
034 import javax.ws.rs.core.MediaType;
035
036 @Path("csv")
037 @Produces(MediaType.APPLICATION_JSON)
038 public class CsvPlugin extends PluginActivator {
039
040 private static Logger log = Logger.getLogger(CsvPlugin.class.getName());
041
042 public static final String FOLDER = "csv";
043
044 public static final char SEPARATOR = '|';
045
046 private boolean isInitialized;
047
048 private FilesService fileService;
049
050 @POST
051 @Path("import/{uri}/{file}")
052 public ImportStatus importCsv(//
053 @PathParam("uri") String typeUri,//
054 @PathParam("file") long fileId,//
055 @HeaderParam("Cookie") ClientState cookie) {
056
057 DeepaMehtaTransaction tx = dms.beginTx();
058 try {
059 List<String[]> lines = readCsvFile(fileId);
060 if (lines.size() < 2) {
061 return new ImportStatus(false, "please upload a valid CSV, see README", null);
062 }
063
064 List<String> childTypeUris = Arrays.asList(lines.get(0));
065 String uriPrefix = childTypeUris.get(0);
066
067 Map<String, Long> topicsByUri = getTopicIdsByUri(typeUri);
068 Map<String, Map<String, Long>> aggrIdsByTypeUriAndValue = getPossibleAggrChilds(typeUri, childTypeUris);
069
070 // status informations
071 int created = 0, deleted = 0, updated = 0;
072
073 // persist each row
074 for (int r = 1; r < lines.size(); r++) {
075 String[] row = lines.get(r);
076 String topicUri = uriPrefix + "." + row[0];
077
078 // create a fresh model
079 TopicModel model = new TopicModel(typeUri);
080 model.setUri(topicUri);
081
082 // map all columns to composite value
083 CompositeValueModel value = new CompositeValueModel();
084 for (int c = 1; c < row.length; c++) {
085 String childTypeUri = childTypeUris.get(c);
086 String childValue = row[c];
087
088 // reference or create a child
089 Map<String, Long> aggrIdsByValue = aggrIdsByTypeUriAndValue.get(childTypeUri);
090 if (aggrIdsByValue != null && aggrIdsByValue.get(childValue) != null) {
091 value.putRef(childTypeUri, aggrIdsByValue.get(childValue));
092 } else {
093 value.put(childTypeUri, childValue);
094 }
095 }
096 model.setCompositeValue(value);
097
098 // create or update a topic
099 Long topicId = topicsByUri.get(topicUri);
100 if (topicId == null) { // create
101 dms.createTopic(model, cookie);
102 created++;
103 } else { // update topic and remove from map
104 model.setId(topicId);
105 topicsByUri.remove(topicUri);
106 dms.updateTopic(model, cookie);
107 updated++;
108 }
109 }
110
111 // delete the remaining instances
112 for (String topicUri : topicsByUri.keySet()) {
113 Long topicId = topicsByUri.get(topicUri);
114 dms.deleteTopic(topicId);
115 deleted++;
116 }
117
118 List<String> status = new ArrayList<String>();
119 status.add("created: " + created);
120 status.add("updated: " + updated);
121 status.add("deleted: " + deleted);
122
123 tx.success();
124 return new ImportStatus(true, "SUCCESS", status);
125 } catch (IOException e) {
126 throw new RuntimeException(e) ;
127 } finally {
128 tx.finish();
129 }
130 }
131
132 /**
133 * get all possible aggregation instances and hash them by typeUri and value
134 *
135 * @param typeUri
136 * @param childTypeUris
137 * @return
138 */
139 private Map<String, Map<String, Long>> getPossibleAggrChilds(String typeUri, List<String> childTypeUris) {
140 TopicType topicType = dms.getTopicType(typeUri);
141 Map<String, Map<String, Long>> aggrIdsByTypeUriAndValue = new HashMap<String, Map<String, Long>>();
142 for (AssociationDefinition associationDefinition : topicType.getAssocDefs()) {
143 if (associationDefinition.getTypeUri().equals("dm4.core.aggregation_def")) {
144 String childTypeUri = associationDefinition.getChildTypeUri();
145 if (childTypeUris.contains(childTypeUri)) {
146 aggrIdsByTypeUriAndValue.put(childTypeUri, getTopicIdsByValue(childTypeUri));
147 }
148 }
149 }
150 return aggrIdsByTypeUriAndValue;
151 }
152
153 /**
154 * get all existing instance topics and hash them by value
155 *
156 * @param childTypeUri
157 * @return instance topics hashed by value
158 */
159 private Map<String, Long> getTopicIdsByValue(String childTypeUri) {
160 Map<String, Long> idsByValue = new HashMap<String, Long>();
161 for (RelatedTopic instance : dms.getTopics(childTypeUri, false, 0).getItems()) {
162 idsByValue.put(instance.getSimpleValue().toString(), instance.getId());
163 }
164 return idsByValue;
165 }
166
167 /**
168 * get all existing instance topics and hash them by URI
169 *
170 * @param typeUri
171 * @return instance topics hashed by URI
172 */
173 private Map<String, Long> getTopicIdsByUri(String typeUri) {
174 Map<String, Long> idsByUri = new HashMap<String, Long>();
175 for (RelatedTopic topic : dms.getTopics(typeUri, false, 0).getItems()) {
176 String topicUri = topic.getUri();
177 if (topicUri != null && topicUri.isEmpty() == false) {
178 idsByUri.put(topicUri, topic.getId());
179 }
180 }
181 return idsByUri;
182 }
183
184 /**
185 * read and validate CSV file
186 *
187 * @param fileId
188 * @return parsed CSV rows with trimmed column array
189 *
190 * @throws FileNotFoundException
191 * @throws IOException
192 */
193 private List<String[]> readCsvFile(long fileId) throws IOException {
194 String fileName = fileService.getFile(fileId).getAbsolutePath();
195 log.info("read CSV " + fileName);
196 CSVReader csvReader = new CSVReader(new FileReader(fileName), SEPARATOR);
197 List<String[]> lines = csvReader.readAll();
198 csvReader.close();
199
200 // trim all columns
201 for (String[] row : lines) {
202 for (int col = 0; col < row.length; col++) {
203 row[col] = row[col].trim();
204 }
205 }
206 return lines;
207 }
208
209 /**
210 * Initialize.
211 */
212 @Override
213 public void init() {
214 isInitialized = true;
215 configureIfReady();
216 }
217
218 @Override
219 @ConsumesService("de.deepamehta.plugins.files.service.FilesService")
220 public void serviceArrived(PluginService service) {
221 if (service instanceof FilesService) {
222 fileService = (FilesService) service;
223 }
224 configureIfReady();
225 }
226
227 @Override
228 public void serviceGone(PluginService service) {
229 if (service == fileService) {
230 fileService = null;
231 }
232 }
233
234 private void configureIfReady() {
235 if (isInitialized && fileService != null) {
236 createCsvDirectory();
237 }
238 }
239
240 private void createCsvDirectory() {
241 // TODO move the initialization to migration "0"
242 try {
243 ResourceInfo resourceInfo = fileService.getResourceInfo(FOLDER);
244 String kind = resourceInfo.toJSON().getString("kind");
245 if (kind.equals("directory") == false) {
246 String repoPath = System.getProperty("dm4.filerepo.path");
247 throw new IllegalStateException("CSV storage directory " + //
248 repoPath + File.separator + FOLDER + " can not be used");
249 }
250 } catch (WebApplicationException e) { // !exists
251 // catch fileService info request error => create directory
252 if (e.getResponse().getStatus() != 404) {
253 throw e;
254 } else {
255 log.info("create CSV directory");
256 fileService.createFolder(FOLDER, "/");
257 }
258 } catch (Exception e) {
259 throw new RuntimeException(e);
260 }
261 }
262 }