001
002 package org.deepamehta.plugins.wikidata;
003
004 import de.deepamehta.core.Association;
005 import de.deepamehta.core.AssociationType;
006 import de.deepamehta.core.RelatedAssociation;
007 import de.deepamehta.core.Topic;
008 import de.deepamehta.core.model.*;
009 import de.deepamehta.core.osgi.PluginActivator;
010 import de.deepamehta.core.service.ClientState;
011 import de.deepamehta.core.service.PluginService;
012 import de.deepamehta.core.service.annotation.ConsumesService;
013 import de.deepamehta.core.storage.spi.DeepaMehtaTransaction;
014 import de.deepamehta.plugins.accesscontrol.service.AccessControlService;
015
016 import java.io.BufferedReader;
017 import java.io.IOException;
018 import java.io.InputStreamReader;
019 import java.net.HttpURLConnection;
020 import java.net.MalformedURLException;
021 import java.net.URL;
022 import java.util.HashMap;
023 import java.util.Iterator;
024 import java.util.List;
025 import java.util.logging.Logger;
026
027 import javax.ws.rs.*;
028 import javax.ws.rs.core.MediaType;
029 import javax.ws.rs.core.Response.Status;
030
031 import org.codehaus.jettison.json.JSONArray;
032 import org.codehaus.jettison.json.JSONException;
033 import org.codehaus.jettison.json.JSONObject;
034 import org.deepamehta.plugins.wikidata.service.WikidataSearchService;
035
036
037
038 /**
039 * A very basic plugin to search and explore wikidata.
040 * Allows to turn a \"Wikidata Search Result Entity\" (of type=property) into DeepaMehta 4 AssociationTypes.
041 *
042 * @author Malte Reißig (<malte@mikromedia.de>)
043 * @website https://github.com/mukil/dm4-wikidata
044 * @version 0.0.3-SNAPSHOT
045 */
046
047 @Path("/wikidata")
048 @Consumes("application/json")
049 @Produces("application/json")
050 public class WikidataSearchPlugin extends PluginActivator implements WikidataSearchService {
051
052 private Logger log = Logger.getLogger(getClass().getName());
053
054 private final String DEEPAMEHTA_VERSION = "DeepaMehta 4.3";
055 private final String WIKIDATA_TYPE_SEARCH_VERSION = "0.0.3-SNAPSHOT";
056 private final String CHARSET = "UTF-8";
057
058 // --- DeepaMehta 4 URIs
059
060 private final String DM_WEBBROWSER_URL = "dm4.webbrowser.url";
061
062 // --- Wikidata DeepaMehta URIs
063
064 private final String WS_WIKIDATA_URI = "org.deepamehta.workspaces.wikidata";
065
066 private final String WD_SEARCH_BUCKET_URI = "org.deepamehta.wikidata.search_bucket";
067 private final String WD_SEARCH_QUERY_URI = "org.deepamehta.wikidata.search_query";
068
069 private final String WD_LANGUAGE_URI = "org.deepamehta.wikidata.language";
070 // private final String WD_LANGUAGE_NAME_URI = "org.deepamehta.wikidata.language_name";
071 // private final String WD_LANGUAGE_ISO_CODE_URI = "org.deepamehta.wikidata.language_code_iso";
072 private final String WD_LANGUAGE_DATA_URI_PREFIX = "org.deepamehta.wikidata.lang_";
073
074 private final String WD_SEARCH_ENTITY_URI = "org.deepamehta.wikidata.search_entity";
075 private final String WD_SEARCH_ENTITY_LABEL_URI = "org.deepamehta.wikidata.search_entity_label";
076 private final String WD_SEARCH_ENTITY_TYPE_URI = "org.deepamehta.wikidata.search_entity_type";
077 private final String WD_SEARCH_ENTITY_ORDINAL_NR = "org.deepamehta.wikidata.search_ordinal_nr";
078 private final String WD_SEARCH_ENTITY_DESCR_URI = "org.deepamehta.wikidata.search_entity_description";
079 private final String WD_SEARCH_ENTITY_ALIAS_URI = "org.deepamehta.wikidata.search_entity_alias";
080 private final String WD_SEARCH_ENTITIY_DATA_URI_PREFIX = "org.deepamehta.wikidata.entity_";
081
082 private final String WD_TEXT_TYPE_URI = "org.deepamehta.wikidata.text";
083 // private final String WD_COMMONS_MEDIA_TYPE_URI = "org.deepamehta.wikidata.commons_media";
084 // private final String WD_GLOBE_COORDINATE_TYPE_URI = "org.deepamehta.wikidata.globe_coordinate";
085
086 private final String WD_ENTITY_CLAIM_EDGE = "org.deepamehta.wikidata.claim_edge";
087
088 // --- Wikidata Service URIs
089
090 private final String WD_SEARCH_ENTITIES_ENDPOINT =
091 "http://www.wikidata.org/w/api.php?action=wbsearchentities&format=json&limit=50";
092 private final String WD_CHECK_ENTITY_CLAIMS_ENDPOINT =
093 "http://www.wikidata.org/w/api.php?action=wbgetclaims&format=json"; // &ungroupedlist=0
094 private final String WD_GET_ENTITY_ENDPOINT = "http://www.wikidata.org/w/api.php?action=wbgetentities"
095 + "&props=info%7Csitelinks%2Furls%7Caliases%7Clabels%7Cdescriptions&dir=ascending&format=json";
096 private final String WD_SEARCH_ENTITY_TYPE_PROPERTY = "property";
097 private final String WD_SEARCH_ENTITY_TYPE_ITEM = "item";
098
099 // --- Instance Variables
100
101 private final String WIKIDATA_ENTITY_URL_PREFIX = "//www.wikidata.org/wiki/";
102 private final String WIKIDATA_PROPERTY_ENTITY_URL_PREFIX = "Property:";
103 private final String WIKIMEDIA_COMMONS_MEDIA_FILE_URL_PREFIX = "//commons.wikimedia.org/wiki/File:";
104
105 private boolean isInitialized = false;
106 private AccessControlService acService = null;
107
108
109
110 @GET
111 @Path("/search/{entity}/{query}/{language_code}")
112 @Produces(MediaType.APPLICATION_JSON)
113 @Override
114 public Topic searchWikidataEntity(@PathParam("query") String query, @PathParam("language_code") String lang,
115 @HeaderParam("Cookie") ClientState clientState, @PathParam("entity") String type) {
116
117 String json_result = "";
118 StringBuffer resultBody = new StringBuffer();
119 URL requestUri = null;
120 Topic search_bucket = null;
121 // sanity check (set en as default-language if nothing was provided by un-initialized language widget)
122 if (lang == null || lang.equals("undefined")) {
123 log.warning("Wikidata Language Search Option was not provided, now requesting data in EN");
124 lang = "en";
125 }
126 // start search operation
127 DeepaMehtaTransaction tx = dms.beginTx();
128 try {
129 // 1) fixme: Authorize request
130 requestUri = new URL(WD_SEARCH_ENTITIES_ENDPOINT + "&search="+ query +"&language="+ lang +"&type=" + type);
131 log.fine("Wikidata Search Entities Request: " + requestUri.toString());
132 // 2) initiate request
133 HttpURLConnection connection = (HttpURLConnection) requestUri.openConnection();
134 connection.setRequestMethod("GET");
135 connection.setRequestProperty("User-Agent", "DeepaMehta "+DEEPAMEHTA_VERSION+" - "
136 + "Wikidata Search " + WIKIDATA_TYPE_SEARCH_VERSION);
137 // 3) check the response
138 int httpStatusCode = connection.getResponseCode();
139 if (httpStatusCode != HttpURLConnection.HTTP_OK) {
140 throw new WebApplicationException(new Throwable("Error with HTTPConnection."),
141 Status.INTERNAL_SERVER_ERROR);
142 }
143 // 4) read in the response
144 BufferedReader rd = new BufferedReader(new InputStreamReader(connection.getInputStream(), CHARSET));
145 for (String input; (input = rd.readLine()) != null;) {
146 resultBody.append(input);
147 }
148 rd.close();
149 // 5) process response
150 if (resultBody.toString().isEmpty()) {
151 throw new WebApplicationException(new RuntimeException("Wikidata was silent."),
152 Status.NO_CONTENT);
153 } else {
154 // ..) Create Wikidata Search Bucket
155 CompositeValueModel bucket_model = new CompositeValueModel();
156 bucket_model.put(WD_SEARCH_QUERY_URI, query);
157 bucket_model.putRef(WD_LANGUAGE_URI, WD_LANGUAGE_DATA_URI_PREFIX + lang);
158 json_result = resultBody.toString();
159 log.fine("Wikidata Search Request Response: " + json_result);
160 processWikidataEntitySearch(json_result, bucket_model, type, lang);
161 search_bucket = dms.createTopic(new TopicModel(WD_SEARCH_BUCKET_URI, bucket_model), clientState);
162 // workaround: addRef does not (yet) fetchComposite, so fetchComposite=true
163 search_bucket = dms.getTopic(search_bucket.getId(), true);
164 log.info("Wikidata Search Bucket for "+ query +" in ("+ lang +") was CREATED");
165 }
166 tx.success();
167 } catch (MalformedURLException e) {
168 log.warning("Wikidata Plugin: MalformedURLException ..." + e.getMessage());
169 tx.failure();
170 throw new RuntimeException("Could not find wikidata endpoint.", e);
171 } catch (IOException ioe) {
172 tx.failure();
173 throw new WebApplicationException(new Throwable(ioe), Status.BAD_REQUEST);
174 } catch (Exception e) {
175 tx.failure();
176 throw new WebApplicationException(new Throwable(e), Status.INTERNAL_SERVER_ERROR);
177 } finally {
178 tx.finish();
179 return search_bucket;
180 }
181 }
182
183 @GET
184 @Path("/{entityId}/{language_code}")
185 @Produces(MediaType.APPLICATION_JSON)
186 @Override
187 public Topic getOrCreateWikidataEntity(@PathParam("entityId") String entityId,
188 @PathParam("language_code") String language_code, @HeaderParam("Cookie") ClientState clientState) {
189 String json_result = "";
190 StringBuffer resultBody = new StringBuffer();
191 URL requestUri = null;
192 Topic entity = null;
193 // sanity check (set en as default-language if nothing was provided by un-initialized language widget)
194 if (language_code == null || language_code.equals("undefined")) {
195 log.warning("Wikidata Language Search Option was not provided, now requesting data in EN");
196 language_code = "en";
197 }
198 try {
199 // 1) fixme: Authorize request
200 // &sites=dewiki&&languages=de
201 requestUri = new URL(WD_GET_ENTITY_ENDPOINT + "&ids="+ entityId + "&languages=" + language_code);
202 log.fine("Requesting Wikidata Entity Details " + requestUri.toString());
203 // 2) initiate request
204 HttpURLConnection connection = (HttpURLConnection) requestUri.openConnection();
205 connection.setRequestMethod("GET");
206 connection.setRequestProperty("User-Agent", "DeepaMehta "+DEEPAMEHTA_VERSION+" - "
207 + "Wikidata Search " + WIKIDATA_TYPE_SEARCH_VERSION);
208 // 3) check the response
209 int httpStatusCode = connection.getResponseCode();
210 if (httpStatusCode != HttpURLConnection.HTTP_OK) {
211 throw new WebApplicationException(new Throwable("Error with HTTPConnection."),
212 Status.INTERNAL_SERVER_ERROR);
213 }
214 // 4) read in the response
215 BufferedReader rd = new BufferedReader(new InputStreamReader(connection.getInputStream(), CHARSET));
216 for (String input; (input = rd.readLine()) != null;) {
217 resultBody.append(input);
218 }
219 rd.close();
220 // 5) process response
221 if (resultBody.toString().isEmpty()) {
222 throw new WebApplicationException(new RuntimeException("Wikidata was silent."),
223 Status.NO_CONTENT);
224 } else {
225 // 6) Create or Update Wikidata Search Entity
226 json_result = resultBody.toString();
227 log.fine("Wikidata Entity Request Response: " + json_result);
228 JSONObject response = new JSONObject(json_result);
229 JSONObject entities = response.getJSONObject("entities");
230 JSONObject response_entity = entities.getJSONObject(entityId);
231 // 0) Check if we need to CREATE or UPDATE our search result entity item
232 Topic existingEntity = dms.getTopic("uri",
233 new SimpleValue(WD_SEARCH_ENTITIY_DATA_URI_PREFIX + entityId), true);
234 if (existingEntity == null) {
235 entity = createWikidataSearchEntity(response_entity, language_code, clientState);
236 } else {
237 // Updates labels, descriptions, aliases, url and (query) language
238 entity = updateWikidataEntity(existingEntity, response_entity, language_code, clientState);
239 }
240 }
241 } catch (MalformedURLException e) {
242 log.warning("Wikidata Plugin: MalformedURLException ..." + e.getMessage());
243 throw new RuntimeException("Could not find wikidata endpoint.", e);
244 } catch (IOException ioe) {
245 throw new WebApplicationException(new Throwable(ioe), Status.BAD_REQUEST);
246 } catch (JSONException je) {
247 throw new WebApplicationException(new Throwable(je), Status.INTERNAL_SERVER_ERROR);
248 } catch (Exception e) {
249 throw new WebApplicationException(new Throwable(e), Status.INTERNAL_SERVER_ERROR);
250 } finally {
251 return entity;
252 }
253 }
254
255 @GET
256 @Path("/check/claims/{id}/{language_code}")
257 @Produces(MediaType.APPLICATION_JSON)
258 @Override
259 public Topic loadClaimsAndRelatedWikidataItems(@PathParam("id") long topicId,
260 @PathParam("language_code") String language_option, @HeaderParam("Cookie") ClientState clientState) {
261
262 String json_result = "";
263 StringBuffer resultBody = new StringBuffer();
264 URL requestUri = null;
265 Topic wikidataItem = dms.getTopic(topicId, true);
266 // 0) sanity check (set en as default-language if nothing was provided by un-initialized language widget)
267 if (language_option == null || language_option.equals("undefined")) {
268 log.warning("Wikidata Language Search Option was not provided, now requesting data in EN.");
269 language_option = "en";
270 }
271 String wikidataId = wikidataItem.getUri().replaceAll(WD_SEARCH_ENTITIY_DATA_URI_PREFIX, "");
272 DeepaMehtaTransaction tx = dms.beginTx();
273 try {
274 // 1) ### Authorize request
275 // 2) ### be explicit and add "&rank=normal" to wbgetclaims-call
276 requestUri = new URL(WD_CHECK_ENTITY_CLAIMS_ENDPOINT + "&entity=" + wikidataId);
277 log.fine("Requesting Wikidata Entity Claims: " + requestUri.toString());
278 // 2) initiate request
279 HttpURLConnection connection = (HttpURLConnection) requestUri.openConnection();
280 connection.setRequestMethod("GET");
281 connection.setRequestProperty("User-Agent", "DeepaMehta "+DEEPAMEHTA_VERSION+" - "
282 + "Wikidata Search " + WIKIDATA_TYPE_SEARCH_VERSION);
283 // 3) check the response
284 int httpStatusCode = connection.getResponseCode();
285 if (httpStatusCode != HttpURLConnection.HTTP_OK) {
286 throw new WebApplicationException(new Throwable("Error with HTTPConnection."),
287 Status.INTERNAL_SERVER_ERROR);
288 }
289 // 4) read in the response
290 BufferedReader rd = new BufferedReader(new InputStreamReader(connection.getInputStream(), CHARSET));
291 for (String input; (input = rd.readLine()) != null;) {
292 resultBody.append(input);
293 }
294 rd.close();
295 // 5) process response
296 if (resultBody.toString().isEmpty()) {
297 throw new WebApplicationException(new RuntimeException("Wikidata was silent."),
298 Status.NO_CONTENT);
299 } else {
300 json_result = resultBody.toString();
301 log.fine("Wikidata Claim Request Response: " + json_result);
302 processWikidataClaims(json_result, wikidataItem, language_option, clientState);
303 log.info("Wikidata Claim Response is FINE");
304 }
305 tx.success();
306 } catch (MalformedURLException e) {
307 log.warning("Wikidata Plugin: MalformedURLException ..." + e.getMessage());
308 tx.failure();
309 throw new RuntimeException("Could not find wikidata endpoint.", e);
310 } catch (IOException ioe) {
311 tx.failure();
312 throw new WebApplicationException(new Throwable(ioe), Status.BAD_REQUEST);
313 } catch (Exception e) {
314 tx.failure();
315 throw new WebApplicationException(new Throwable(e), Status.INTERNAL_SERVER_ERROR);
316 } finally {
317 tx.finish();
318 return wikidataItem;
319 }
320 }
321
322 @GET
323 @Path("/property/turn/{id}")
324 @Produces(MediaType.APPLICATION_JSON)
325 @Override
326 public Topic createWikidataAssociationType(@PathParam("id") long id,
327 @HeaderParam("Cookie") ClientState clientState) {
328
329 AssociationType association_type = null;
330 DeepaMehtaTransaction tx = dms.beginTx();
331 try {
332 Topic property_entity = dms.getTopic(id, true);
333 // 1) Create new Association Type model
334 String property_name = property_entity.getSimpleValue().toString();
335 AssociationTypeModel assoc_type_model = new AssociationTypeModel("org.deepamehta.wikidata.assoctype_"
336 + property_entity.getUri().replaceAll(WD_SEARCH_ENTITIY_DATA_URI_PREFIX, ""),
337 property_name, "dm4.core.text");
338 association_type = dms.createAssociationType(assoc_type_model, clientState);
339 // 2) Assign to "Wikidata" Workspace
340 assignToWikidataWorkspace(association_type);
341 // 3) Associated search-result-entity to new assoc-type (to keep track)
342 dms.createAssociation(new AssociationModel("dm4.core.association",
343 new TopicRoleModel(property_entity.getUri(), "dm4.core.default"),
344 new TopicRoleModel(association_type.getUri(), "dm4.core.default")
345 ), clientState);
346 log.info("Turned wikidata property \""+ property_entity.getUri() +"\" into DM Association Type!");
347 tx.success();
348 } catch (Error e) {
349 tx.failure();
350 log.warning("OH: The Wikidata Plugin experienced an unforeseen error! "+ e.getMessage());
351 } finally {
352 tx.finish();
353 return association_type;
354 }
355 }
356
357 @GET
358 @Path("/property/related/claims/{id}")
359 @Produces(MediaType.APPLICATION_JSON)
360 @Override
361 public List<RelatedAssociation> getTopicRelatedAssociations (@PathParam("id") long topicId) {
362 Topic topic = dms.getTopic(topicId, false);
363 List<RelatedAssociation> associations = topic.getRelatedAssociations("dm4.core.aggregation",
364 "dm4.core.child", "dm4.core.parent", "org.deepamehta.wikidata.claim_edge", false, false);
365 return associations;
366 }
367
368
369 // --
370 // --- Wikidata Search (Application Specific) Private Methods
371 // --
372
373 private void processWikidataEntitySearch(String json_result, CompositeValueModel search_bucket,
374 String type, String lang) {
375 try {
376 JSONObject response = new JSONObject(json_result);
377 JSONArray result = response.getJSONArray("search");
378 if (result.length() > 0) {
379 for (int i = 0; i < result.length(); i++) {
380 JSONObject entity_response = result.getJSONObject(i);
381 // Check if entity already exists
382 String id = entity_response.getString("id");
383 Topic existing_entity = dms.getTopic("uri",
384 new SimpleValue(WD_SEARCH_ENTITIY_DATA_URI_PREFIX + id), false);
385 if (existing_entity == null) {
386 // Create new search entity composite
387 String name = entity_response.getString("label");
388 String url = entity_response.getString("url");
389 //
390 CompositeValueModel entity_composite = new CompositeValueModel();
391 entity_composite.put(WD_SEARCH_ENTITY_LABEL_URI, name);
392 if (entity_response.has("description")) {
393 String description = entity_response.getString("description");
394 entity_composite.put(WD_SEARCH_ENTITY_DESCR_URI, description);
395 }
396 entity_composite.put(DM_WEBBROWSER_URL, url);
397 if (entity_response.has("aliases")) {
398 JSONArray aliases = entity_response.getJSONArray("aliases");
399 for (int a=0; a < aliases.length(); a++) {
400 String alias = aliases.getString(a);
401 entity_composite.add(WD_SEARCH_ENTITY_ALIAS_URI,
402 new TopicModel(WD_SEARCH_ENTITY_ALIAS_URI, new SimpleValue(alias)));
403 }
404 }
405 // set enity place in resultset
406 entity_composite.put(WD_SEARCH_ENTITY_ORDINAL_NR, i);
407 // set entity-type
408 entity_composite.put(WD_SEARCH_ENTITY_TYPE_URI, type);
409 // set language-value on entity-result
410 entity_composite.putRef(WD_LANGUAGE_URI, WD_LANGUAGE_DATA_URI_PREFIX + lang);
411 TopicModel entity_model = new TopicModel(WD_SEARCH_ENTITIY_DATA_URI_PREFIX + id,
412 WD_SEARCH_ENTITY_URI, entity_composite);
413 // create and reference entity in wikidata search bucket
414 search_bucket.add(WD_SEARCH_ENTITY_URI, entity_model);
415 } else {
416 // reference existing entity in wikidata search bucket by URI
417 search_bucket.addRef(WD_SEARCH_ENTITY_URI, WD_SEARCH_ENTITIY_DATA_URI_PREFIX + id);
418 }
419 }
420 }
421 } catch (JSONException ex) {
422 log.warning("Wikidata Plugin: JSONException during processing a wikidata entity search response. "
423 + ex.getMessage());
424 }
425 }
426
427 private Topic createWikidataSearchEntity(JSONObject entity_response, String lang, ClientState clientState) {
428 Topic entity = null;
429 DeepaMehtaTransaction tx = dms.beginTx();
430 try {
431 String id = entity_response.getString("id");
432 // Create new search entity composite
433 CompositeValueModel entity_composite = buildWikidataEntityModel(entity_response, lang);
434 TopicModel entity_model = new TopicModel(WD_SEARCH_ENTITIY_DATA_URI_PREFIX + id,
435 WD_SEARCH_ENTITY_URI, entity_composite);
436 entity = dms.createTopic(entity_model, clientState);
437 log.fine("Wikidata Search Entity Created (" +
438 entity_composite.getString(WD_SEARCH_ENTITY_TYPE_URI)+ "): \"" + entity.getSimpleValue() +"\" - FINE!");
439 tx.success();
440 } catch (Exception ex) {
441 log.warning("FAILED to create a \"Wikidata Search Entity\" caused by " + ex.getMessage());
442 tx.failure();
443 } finally {
444 tx.finish();
445 return entity;
446 }
447 }
448
449 private Topic updateWikidataEntity(Topic entity, JSONObject entity_response, String lang, ClientState clientState) {
450 DeepaMehtaTransaction tx = dms.beginTx();
451 try {
452 // Update existing search entity topic
453 CompositeValueModel entity_composite = buildWikidataEntityModel(entity_response, lang);
454 TopicModel entity_model = new TopicModel(entity.getId(), entity_composite);
455 dms.updateTopic(entity_model, clientState);
456 log.fine("Wikidata Search Entity Updated (" +
457 entity_composite.getString(WD_SEARCH_ENTITY_TYPE_URI)+ "): \"" + entity.getSimpleValue() +"\" - FINE!");
458 tx.success();
459 } catch (Exception ex) {
460 log.warning("FAILED to UPDATE \"Wikidata Search Entity\" caused by " + ex.getMessage());
461 tx.failure();
462 } finally {
463 tx.finish();
464 return entity;
465 }
466 }
467
468 private CompositeValueModel buildWikidataEntityModel(JSONObject entity_response, String lang) {
469 CompositeValueModel entity_composite = new CompositeValueModel();
470 try {
471 String id = entity_response.getString("id");
472 String type = entity_response.getString("type");
473 entity_composite = new CompositeValueModel();
474 // main label
475 if (entity_response.has("labels")) {
476 JSONObject labels = entity_response.getJSONObject("labels");
477 JSONObject languaged_label = labels.getJSONObject(lang);
478 String label = languaged_label.getString("value");
479 entity_composite.put(WD_SEARCH_ENTITY_LABEL_URI, label);
480 }
481 // main description
482 if (entity_response.has("descriptions")) {
483 JSONObject descriptions = entity_response.getJSONObject("descriptions");
484 JSONObject languaged_descr = descriptions.getJSONObject(lang);
485 String description = languaged_descr.getString("value");
486 entity_composite.put(WD_SEARCH_ENTITY_DESCR_URI, description);
487 }
488 // aliases
489 if (entity_response.has("aliases")) {
490 JSONObject aliases = entity_response.getJSONObject("aliases");
491 JSONArray languaged_aliases = aliases.getJSONArray(lang);
492 for (int a=0; a < languaged_aliases.length(); a++) {
493 JSONObject alias_object = languaged_aliases.getJSONObject(a);
494 String alias = alias_object.getString("value");
495 entity_composite.add(WD_SEARCH_ENTITY_ALIAS_URI,
496 new TopicModel(WD_SEARCH_ENTITY_ALIAS_URI, new SimpleValue(alias)));
497 }
498 }
499 // set wikidata url
500 if (type.equals(WD_SEARCH_ENTITY_TYPE_PROPERTY)) {
501 entity_composite.put(DM_WEBBROWSER_URL, WIKIDATA_ENTITY_URL_PREFIX
502 + WIKIDATA_PROPERTY_ENTITY_URL_PREFIX + id);
503 } else {
504 entity_composite.put(DM_WEBBROWSER_URL, WIKIDATA_ENTITY_URL_PREFIX + id);
505 }
506 // set language-value on entity-result
507 entity_composite.putRef(WD_LANGUAGE_URI, WD_LANGUAGE_DATA_URI_PREFIX + lang);
508 // ### sitelinks
509 /** if (entity_response.has("sitelinks")) {
510 JSONObject sitelinks = entity_response.getJSONObject("sitelinks");
511 if (sitelinks.has(lang + "wiki")) {
512 JSONObject sitelink = sitelinks.getJSONObject(lang + "wiki");
513 entity_composite.put(DM_WEBBROWSER_URL, sitelink.getString("url"));
514 } else {
515 log.warning("There is no sitelink for this item in this language/wiki: " + lang + "wiki");
516 }
517 } **/
518 entity_composite.put(WD_SEARCH_ENTITY_TYPE_URI, type);
519 } catch (JSONException jex) {
520 log.warning("JSONException during build up of the search-entities composite model");
521 return null;
522 } finally {
523 return entity_composite;
524 }
525 }
526
527
528 private void processWikidataClaims(String json_result, Topic wikidataItem, String language_code,
529 ClientState clientState) {
530 try {
531 JSONObject response = new JSONObject(json_result);
532 JSONObject result = response.getJSONObject("claims");
533 // ### Needs to identify if claims (already imported in DM4) are not yet part of the current wikidata-data
534 Iterator properties = result.keys();
535 log.info("Wikidata Plugin is processing all properties part of related CLAIMS");
536 Topic propertyEntity = null;
537 while (properties.hasNext()) {
538 String property_id = properties.next().toString();
539 // 1) Load related property-entity
540 propertyEntity = getOrCreateWikidataEntity(property_id, language_code, clientState);
541 // HashMap<String, List<Topic>> all_entities = new HashMap<String, List<Topic>>();
542 JSONArray property_listing = result.getJSONArray(property_id);
543 for (int i=0; i < property_listing.length(); i++) {
544 // 2) fetch related wikidata entity
545 Topic referencedItemEntity = null;
546 JSONObject entity_response = property_listing.getJSONObject(i);
547 JSONObject mainsnak = entity_response.getJSONObject("mainsnak");
548 String claim_guid = entity_response.getString("id");
549 // 3) build up item as part of the claim (if so)
550 String itemId = "";
551 String snakDataType = mainsnak.getString("datatype");
552 JSONObject snakDataValue = mainsnak.getJSONObject("datavalue");
553 // ..) depending on the various (claimed/realted) value-types
554 if (snakDataType.equals("wikibase-item")) {
555 // log.info("Wikibase Item claimed via \"" + propertyEntity.getSimpleValue() + "\"");
556 JSONObject snakDataValueValue = snakDataValue.getJSONObject("value");
557 long numericId = snakDataValueValue.getLong("numeric-id");
558 itemId = "Q" + numericId; // is this always of entity-type "item"? responses looks like.
559 referencedItemEntity = getOrCreateWikidataEntity(itemId, language_code, clientState);
560 } else if (snakDataType.equals("commonsMedia")) {
561 // do relate wikidata.commons_media
562 log.fine("Commons Media claimed via \"" + propertyEntity.getSimpleValue()
563 + "\" ("+language_code+") DEBUG:");
564 log.fine(" " + snakDataValue.toString());
565 // ### make use of WIKIMEDIA_COMMONS_MEDIA_FILE_URL_PREFIX and implement page-renderer
566 } else if (snakDataType.equals("globe-coordinate")) {
567 // do relate wikidata.globe_coordinate
568 log.fine("Globe Coordinate claimed via \"" + propertyEntity.getSimpleValue()
569 + "\" ("+language_code+") DEBUG:");
570 log.fine(" " + snakDataValue.toString());
571 } else if (snakDataType.equals("string")) {
572 if (snakDataValue.has("value")) {
573 String value = snakDataValue.getString("value");
574 referencedItemEntity = getOrCreateWikidataText(value, language_code, clientState);
575 } else {
576 log.warning("Could not access wikidata-text value - json-response EMPTY!");
577 }
578 } else {
579 log.warning("Value claimed as " + propertyEntity.getSimpleValue() + " is not of any known type"
580 + " wikibase-item but \"" + snakDataType +"\" ("+snakDataValue+")");
581 // e.g. snakDataType.equals("quantity")
582 }
583 // store topic reference to (new or already existing) wikidata-entity/ resp. -value topic
584 if (referencedItemEntity != null) {
585 createWikidataClaimEdge(claim_guid, wikidataItem, referencedItemEntity,
586 propertyEntity, clientState);
587 } else {
588 log.warning("SKIPPED creating claim of type \""+snakDataType+"\" value for "
589 + "\""+propertyEntity.getSimpleValue()+"\"");
590 }
591 }
592 /** Iterator entity_iterator = all_entities.keySet().iterator();
593 StringBuffer requesting_ids = new StringBuffer();
594 while (entity_iterator.hasNext()) {
595 String entity_id = entity_iterator.next().toString();
596 requesting_ids.append(entity_id + "|");
597 }
598 log.info("Requesting ALL ITEMS for " +property_id+ ": " + requesting_ids.toString());
599 omitting this solution bcause: "*": "Too many values supplied for parameter 'ids': the limit is 50" **/
600 }
601 } catch (JSONException ex) {
602 log.warning("JSONException during processing a wikidata claim. " + ex.getMessage());
603 }
604 }
605
606 private Association createWikidataClaimEdge (String claim_guid, Topic one, Topic two,
607 Topic property, ClientState clientState) {
608 DeepaMehtaTransaction tx = dms.beginTx();
609 Association claim = null;
610 try {
611 if (!associationExists(WD_ENTITY_CLAIM_EDGE, one, two)) {
612 // 1) Create \"Wikidata Claim\"-Edge with GUID
613 claim = dms.createAssociation(new AssociationModel(WD_ENTITY_CLAIM_EDGE,
614 new TopicRoleModel(one.getId(), "dm4.core.default"),
615 new TopicRoleModel(two.getId(), "dm4.core.default")), clientState);
616 claim.setUri(claim_guid);
617 log.info("Created \"Wikidata Claim\" with GUID: " + claim.getUri() +" for \"" + two.getSimpleValue() +
618 " (property: " + property.getSimpleValue() +
619 "\") for \"" + one.getSimpleValue() + "\" - FINE");
620 // 2) Assign wikidata property (=Wikidata Search Entity) to this claim-edge
621 claim.setCompositeValue(new CompositeValueModel().putRef(WD_SEARCH_ENTITY_URI,
622 property.getUri()), clientState, null);
623 }
624 tx.success();
625 } catch (Exception e) {
626 log.severe("FAILED to create a \"Claim\" between \""+one.getSimpleValue()+"\" - \""+two.getSimpleValue());
627 tx.failure();
628 throw new RuntimeException(e);
629 } finally {
630 tx.finish();
631 return claim;
632 }
633 }
634
635 private Topic getOrCreateWikidataText(String value, String lang, ClientState clientState) {
636 Topic textValue = null;
637 // 1) query for text-value
638 log.info("Value: " + value);
639 try {
640 textValue = dms.getTopic(WD_TEXT_TYPE_URI, new SimpleValue(value), false);
641 } catch (Exception ex) {
642 log.info("Could not find a wikidata-text value topic for " + value + ex.getMessage());
643 }
644 // 2) re-use or create
645 DeepaMehtaTransaction tx = dms.beginTx();
646 try {
647 if (textValue == null) {
648 textValue = dms.createTopic(new TopicModel(WD_TEXT_TYPE_URI, new SimpleValue(value)), clientState);
649 log.info("CREATED \"Wikidata Text\" - \"" + value +"\" (" + lang + ") - OK!");
650 } else {
651 log.info("FETCHED \"Wikidata Text\" - \"" + textValue.getSimpleValue() +"\" "
652 + "(" + lang + ") - Re-using it!");
653 }
654 tx.success();
655 } catch (Exception ex) {
656 log.warning("FAILURE during creating a wikidata value topic: " + ex.getLocalizedMessage());
657 tx.failure();
658 } finally {
659 tx.finish();
660 return textValue;
661 }
662 }
663
664
665
666 // --
667 // --- DeepaMehta 4 Plugin Related Private Methods
668 // --
669
670 private boolean associationExists(String edge_type, Topic item, Topic user) {
671 List<Association> results = dms.getAssociations(item.getId(), user.getId(), edge_type);
672 return (results.size() > 0) ? true : false;
673 }
674
675 private void assignToWikidataWorkspace(Topic topic) {
676 // fixme: remove assignment of type to other (selected via clientState) workspace
677 Topic defaultWorkspace = dms.getTopic("uri", new SimpleValue(WS_WIKIDATA_URI), false);
678 dms.createAssociation(new AssociationModel("dm4.core.aggregation",
679 new TopicRoleModel(topic.getId(), "dm4.core.parent"),
680 new TopicRoleModel(defaultWorkspace.getId(), "dm4.core.child")
681 ), null);
682 }
683
684 /** --- Implementing PluginService Interfaces to consume AccessControlService --- */
685
686 @Override
687 @ConsumesService({
688 "de.deepamehta.plugins.accesscontrol.service.AccessControlService"
689 })
690 public void serviceArrived(PluginService service) {
691 if (service instanceof AccessControlService) {
692 acService = (AccessControlService) service;
693 }
694 }
695
696 @Override
697 @ConsumesService({
698 "de.deepamehta.plugins.accesscontrol.service.AccessControlService"
699 })
700 public void serviceGone(PluginService service) {
701 if (service == acService) {
702 acService = null;
703 }
704 }
705
706 }