001    
002    package org.deepamehta.plugins.wikidata;
003    
004    import de.deepamehta.core.Association;
005    import de.deepamehta.core.AssociationType;
006    import de.deepamehta.core.RelatedAssociation;
007    import de.deepamehta.core.Topic;
008    import de.deepamehta.core.model.*;
009    import de.deepamehta.core.osgi.PluginActivator;
010    import de.deepamehta.core.service.ClientState;
011    import de.deepamehta.core.service.PluginService;
012    import de.deepamehta.core.service.annotation.ConsumesService;
013    import de.deepamehta.core.storage.spi.DeepaMehtaTransaction;
014    import de.deepamehta.plugins.accesscontrol.service.AccessControlService;
015    
016    import java.io.BufferedReader;
017    import java.io.IOException;
018    import java.io.InputStreamReader;
019    import java.net.HttpURLConnection;
020    import java.net.MalformedURLException;
021    import java.net.URL;
022    import java.util.HashMap;
023    import java.util.Iterator;
024    import java.util.List;
025    import java.util.logging.Logger;
026    
027    import javax.ws.rs.*;
028    import javax.ws.rs.core.MediaType;
029    import javax.ws.rs.core.Response.Status;
030    
031    import org.codehaus.jettison.json.JSONArray;
032    import org.codehaus.jettison.json.JSONException;
033    import org.codehaus.jettison.json.JSONObject;
034    import org.deepamehta.plugins.wikidata.service.WikidataSearchService;
035    
036    
037    
038    /**
039     * A very basic plugin to search and explore wikidata.
040     * Allows to turn a \"Wikidata Search Result Entity\" (of type=property) into DeepaMehta 4 AssociationTypes.
041     *
042     * @author Malte Rei├čig (<malte@mikromedia.de>)
043     * @website https://github.com/mukil/dm4-wikidata
044     * @version 0.0.3-SNAPSHOT
045     */
046    
047    @Path("/wikidata")
048    @Consumes("application/json")
049    @Produces("application/json")
050    public class WikidataSearchPlugin extends PluginActivator implements WikidataSearchService {
051    
052        private Logger log = Logger.getLogger(getClass().getName());
053    
054        private final String DEEPAMEHTA_VERSION = "DeepaMehta 4.3";
055        private final String WIKIDATA_TYPE_SEARCH_VERSION = "0.0.3-SNAPSHOT";
056        private final String CHARSET = "UTF-8";
057    
058        // --- DeepaMehta 4 URIs
059    
060        private final String DM_WEBBROWSER_URL = "dm4.webbrowser.url";
061    
062        // --- Wikidata DeepaMehta URIs
063    
064        private final String WS_WIKIDATA_URI = "org.deepamehta.workspaces.wikidata";
065    
066        private final String WD_SEARCH_BUCKET_URI = "org.deepamehta.wikidata.search_bucket";
067        private final String WD_SEARCH_QUERY_URI = "org.deepamehta.wikidata.search_query";
068    
069        private final String WD_LANGUAGE_URI = "org.deepamehta.wikidata.language";
070        // private final String WD_LANGUAGE_NAME_URI = "org.deepamehta.wikidata.language_name";
071        // private final String WD_LANGUAGE_ISO_CODE_URI = "org.deepamehta.wikidata.language_code_iso";
072        private final String WD_LANGUAGE_DATA_URI_PREFIX = "org.deepamehta.wikidata.lang_";
073    
074        private final String WD_SEARCH_ENTITY_URI = "org.deepamehta.wikidata.search_entity";
075        private final String WD_SEARCH_ENTITY_LABEL_URI = "org.deepamehta.wikidata.search_entity_label";
076        private final String WD_SEARCH_ENTITY_TYPE_URI = "org.deepamehta.wikidata.search_entity_type";
077        private final String WD_SEARCH_ENTITY_ORDINAL_NR = "org.deepamehta.wikidata.search_ordinal_nr";
078        private final String WD_SEARCH_ENTITY_DESCR_URI = "org.deepamehta.wikidata.search_entity_description";
079        private final String WD_SEARCH_ENTITY_ALIAS_URI = "org.deepamehta.wikidata.search_entity_alias";
080        private final String WD_SEARCH_ENTITIY_DATA_URI_PREFIX = "org.deepamehta.wikidata.entity_";
081    
082        private final String WD_TEXT_TYPE_URI = "org.deepamehta.wikidata.text";
083        // private final String WD_COMMONS_MEDIA_TYPE_URI = "org.deepamehta.wikidata.commons_media";
084        // private final String WD_GLOBE_COORDINATE_TYPE_URI = "org.deepamehta.wikidata.globe_coordinate";
085    
086        private final String WD_ENTITY_CLAIM_EDGE = "org.deepamehta.wikidata.claim_edge";
087    
088        // --- Wikidata Service URIs
089    
090        private final String WD_SEARCH_ENTITIES_ENDPOINT =
091                "http://www.wikidata.org/w/api.php?action=wbsearchentities&format=json&limit=50";
092        private final String WD_CHECK_ENTITY_CLAIMS_ENDPOINT =
093                "http://www.wikidata.org/w/api.php?action=wbgetclaims&format=json"; // &ungroupedlist=0
094        private final String WD_GET_ENTITY_ENDPOINT = "http://www.wikidata.org/w/api.php?action=wbgetentities"
095                + "&props=info%7Csitelinks%2Furls%7Caliases%7Clabels%7Cdescriptions&dir=ascending&format=json";
096        private final String WD_SEARCH_ENTITY_TYPE_PROPERTY = "property";
097        private final String WD_SEARCH_ENTITY_TYPE_ITEM = "item";
098    
099        // --- Instance Variables
100    
101        private final String WIKIDATA_ENTITY_URL_PREFIX = "//www.wikidata.org/wiki/";
102        private final String WIKIDATA_PROPERTY_ENTITY_URL_PREFIX = "Property:";
103        private final String WIKIMEDIA_COMMONS_MEDIA_FILE_URL_PREFIX = "//commons.wikimedia.org/wiki/File:";
104    
105        private boolean isInitialized = false;
106        private AccessControlService acService = null;
107    
108    
109    
110        @GET
111        @Path("/search/{entity}/{query}/{language_code}")
112        @Produces(MediaType.APPLICATION_JSON)
113        @Override
114        public Topic searchWikidataEntity(@PathParam("query") String query, @PathParam("language_code") String lang,
115                    @HeaderParam("Cookie") ClientState clientState, @PathParam("entity") String type) {
116    
117            String json_result = "";
118            StringBuffer resultBody = new StringBuffer();
119            URL requestUri = null;
120            Topic search_bucket = null;
121            // sanity check (set en as default-language if nothing was provided by un-initialized language widget)
122            if (lang == null || lang.equals("undefined")) {
123                log.warning("Wikidata Language Search Option was not provided, now requesting data in EN");
124                lang = "en";
125            }
126            // start search operation
127            DeepaMehtaTransaction tx = dms.beginTx();
128            try {
129                // 1) fixme: Authorize request
130                requestUri = new URL(WD_SEARCH_ENTITIES_ENDPOINT + "&search="+ query +"&language="+ lang +"&type=" + type);
131                log.fine("Wikidata Search Entities Request: " + requestUri.toString());
132                // 2) initiate request
133                HttpURLConnection connection = (HttpURLConnection) requestUri.openConnection();
134                connection.setRequestMethod("GET");
135                connection.setRequestProperty("User-Agent", "DeepaMehta "+DEEPAMEHTA_VERSION+" - "
136                        + "Wikidata Search " + WIKIDATA_TYPE_SEARCH_VERSION);
137                // 3) check the response
138                int httpStatusCode = connection.getResponseCode();
139                if (httpStatusCode != HttpURLConnection.HTTP_OK) {
140                    throw new WebApplicationException(new Throwable("Error with HTTPConnection."),
141                            Status.INTERNAL_SERVER_ERROR);
142                }
143                // 4) read in the response
144                BufferedReader rd = new BufferedReader(new InputStreamReader(connection.getInputStream(), CHARSET));
145                for (String input; (input = rd.readLine()) != null;) {
146                    resultBody.append(input);
147                }
148                rd.close();
149                // 5) process response
150                if (resultBody.toString().isEmpty()) {
151                    throw new WebApplicationException(new RuntimeException("Wikidata was silent."),
152                            Status.NO_CONTENT);
153                } else {
154                    // ..) Create Wikidata Search Bucket
155                    CompositeValueModel bucket_model = new CompositeValueModel();
156                    bucket_model.put(WD_SEARCH_QUERY_URI, query);
157                    bucket_model.putRef(WD_LANGUAGE_URI, WD_LANGUAGE_DATA_URI_PREFIX + lang);
158                    json_result = resultBody.toString();
159                    log.fine("Wikidata Search Request Response: " + json_result);
160                    processWikidataEntitySearch(json_result, bucket_model, type, lang);
161                    search_bucket = dms.createTopic(new TopicModel(WD_SEARCH_BUCKET_URI, bucket_model), clientState);
162                    // workaround: addRef does not (yet) fetchComposite, so fetchComposite=true
163                    search_bucket = dms.getTopic(search_bucket.getId(), true);
164                    log.info("Wikidata Search Bucket for "+ query +" in ("+ lang +") was CREATED");
165                }
166                tx.success();
167            } catch (MalformedURLException e) {
168                log.warning("Wikidata Plugin: MalformedURLException ..." + e.getMessage());
169                tx.failure();
170                throw new RuntimeException("Could not find wikidata endpoint.", e);
171            } catch (IOException ioe) {
172                tx.failure();
173                throw new WebApplicationException(new Throwable(ioe), Status.BAD_REQUEST);
174            } catch (Exception e) {
175                tx.failure();
176                throw new WebApplicationException(new Throwable(e), Status.INTERNAL_SERVER_ERROR);
177            } finally {
178                tx.finish();
179                return search_bucket;
180            }
181        }
182    
183        @GET
184        @Path("/{entityId}/{language_code}")
185        @Produces(MediaType.APPLICATION_JSON)
186        @Override
187        public Topic getOrCreateWikidataEntity(@PathParam("entityId") String entityId,
188            @PathParam("language_code") String language_code, @HeaderParam("Cookie") ClientState clientState) {
189            String json_result = "";
190            StringBuffer resultBody = new StringBuffer();
191            URL requestUri = null;
192            Topic entity = null;
193            // sanity check (set en as default-language if nothing was provided by un-initialized language widget)
194            if (language_code == null || language_code.equals("undefined")) {
195                log.warning("Wikidata Language Search Option was not provided, now requesting data in EN");
196                language_code = "en";
197            }
198            try {
199                // 1) fixme: Authorize request
200                // &sites=dewiki&&languages=de
201                requestUri = new URL(WD_GET_ENTITY_ENDPOINT + "&ids="+ entityId + "&languages=" + language_code);
202                log.fine("Requesting Wikidata Entity Details " + requestUri.toString());
203                // 2) initiate request
204                HttpURLConnection connection = (HttpURLConnection) requestUri.openConnection();
205                connection.setRequestMethod("GET");
206                connection.setRequestProperty("User-Agent", "DeepaMehta "+DEEPAMEHTA_VERSION+" - "
207                        + "Wikidata Search " + WIKIDATA_TYPE_SEARCH_VERSION);
208                // 3) check the response
209                int httpStatusCode = connection.getResponseCode();
210                if (httpStatusCode != HttpURLConnection.HTTP_OK) {
211                    throw new WebApplicationException(new Throwable("Error with HTTPConnection."),
212                            Status.INTERNAL_SERVER_ERROR);
213                }
214                // 4) read in the response
215                BufferedReader rd = new BufferedReader(new InputStreamReader(connection.getInputStream(), CHARSET));
216                for (String input; (input = rd.readLine()) != null;) {
217                    resultBody.append(input);
218                }
219                rd.close();
220                // 5) process response
221                if (resultBody.toString().isEmpty()) {
222                    throw new WebApplicationException(new RuntimeException("Wikidata was silent."),
223                            Status.NO_CONTENT);
224                } else {
225                    // 6) Create or Update Wikidata Search Entity
226                    json_result = resultBody.toString();
227                    log.fine("Wikidata Entity Request Response: " + json_result);
228                    JSONObject response = new JSONObject(json_result);
229                    JSONObject entities = response.getJSONObject("entities");
230                    JSONObject response_entity = entities.getJSONObject(entityId);
231                    // 0) Check if we need to CREATE or UPDATE our search result entity item
232                    Topic existingEntity = dms.getTopic("uri",
233                                new SimpleValue(WD_SEARCH_ENTITIY_DATA_URI_PREFIX + entityId), true);
234                    if (existingEntity == null) {
235                        entity = createWikidataSearchEntity(response_entity, language_code, clientState);
236                    } else {
237                        // Updates labels, descriptions, aliases, url and (query) language
238                        entity = updateWikidataEntity(existingEntity, response_entity, language_code, clientState);
239                    }
240                }
241            } catch (MalformedURLException e) {
242                log.warning("Wikidata Plugin: MalformedURLException ..." + e.getMessage());
243                throw new RuntimeException("Could not find wikidata endpoint.", e);
244            } catch (IOException ioe) {
245                throw new WebApplicationException(new Throwable(ioe), Status.BAD_REQUEST);
246            } catch (JSONException je) {
247                throw new WebApplicationException(new Throwable(je), Status.INTERNAL_SERVER_ERROR);
248            } catch (Exception e) {
249                throw new WebApplicationException(new Throwable(e), Status.INTERNAL_SERVER_ERROR);
250            } finally {
251                return entity;
252            }
253        }
254    
255        @GET
256        @Path("/check/claims/{id}/{language_code}")
257        @Produces(MediaType.APPLICATION_JSON)
258        @Override
259        public Topic loadClaimsAndRelatedWikidataItems(@PathParam("id") long topicId,
260                @PathParam("language_code") String language_option, @HeaderParam("Cookie") ClientState clientState) {
261    
262            String json_result = "";
263            StringBuffer resultBody = new StringBuffer();
264            URL requestUri = null;
265            Topic wikidataItem = dms.getTopic(topicId, true);
266            // 0) sanity check (set en as default-language if nothing was provided by un-initialized language widget)
267            if (language_option == null || language_option.equals("undefined")) {
268                log.warning("Wikidata Language Search Option was not provided, now requesting data in EN.");
269                language_option = "en";
270            }
271            String wikidataId = wikidataItem.getUri().replaceAll(WD_SEARCH_ENTITIY_DATA_URI_PREFIX, "");
272            DeepaMehtaTransaction tx = dms.beginTx();
273            try {
274                // 1) ### Authorize request
275                // 2) ### be explicit and add "&rank=normal" to wbgetclaims-call
276                requestUri = new URL(WD_CHECK_ENTITY_CLAIMS_ENDPOINT + "&entity=" + wikidataId);
277                log.fine("Requesting Wikidata Entity Claims: " + requestUri.toString());
278                // 2) initiate request
279                HttpURLConnection connection = (HttpURLConnection) requestUri.openConnection();
280                connection.setRequestMethod("GET");
281                connection.setRequestProperty("User-Agent", "DeepaMehta "+DEEPAMEHTA_VERSION+" - "
282                        + "Wikidata Search " + WIKIDATA_TYPE_SEARCH_VERSION);
283                // 3) check the response
284                int httpStatusCode = connection.getResponseCode();
285                if (httpStatusCode != HttpURLConnection.HTTP_OK) {
286                    throw new WebApplicationException(new Throwable("Error with HTTPConnection."),
287                            Status.INTERNAL_SERVER_ERROR);
288                }
289                // 4) read in the response
290                BufferedReader rd = new BufferedReader(new InputStreamReader(connection.getInputStream(), CHARSET));
291                for (String input; (input = rd.readLine()) != null;) {
292                    resultBody.append(input);
293                }
294                rd.close();
295                // 5) process response
296                if (resultBody.toString().isEmpty()) {
297                    throw new WebApplicationException(new RuntimeException("Wikidata was silent."),
298                            Status.NO_CONTENT);
299                } else {
300                    json_result = resultBody.toString();
301                    log.fine("Wikidata Claim Request Response: " + json_result);
302                    processWikidataClaims(json_result, wikidataItem, language_option, clientState);
303                    log.info("Wikidata Claim Response is FINE");
304                }
305                tx.success();
306            } catch (MalformedURLException e) {
307                log.warning("Wikidata Plugin: MalformedURLException ..." + e.getMessage());
308                tx.failure();
309                throw new RuntimeException("Could not find wikidata endpoint.", e);
310            } catch (IOException ioe) {
311                tx.failure();
312                throw new WebApplicationException(new Throwable(ioe), Status.BAD_REQUEST);
313            } catch (Exception e) {
314                tx.failure();
315                throw new WebApplicationException(new Throwable(e), Status.INTERNAL_SERVER_ERROR);
316            } finally {
317                tx.finish();
318                return wikidataItem;
319            }
320        }
321    
322        @GET
323        @Path("/property/turn/{id}")
324        @Produces(MediaType.APPLICATION_JSON)
325        @Override
326        public Topic createWikidataAssociationType(@PathParam("id") long id,
327                @HeaderParam("Cookie") ClientState clientState) {
328    
329            AssociationType association_type = null;
330            DeepaMehtaTransaction tx = dms.beginTx();
331            try {
332                Topic property_entity = dms.getTopic(id, true);
333                // 1) Create new Association Type model
334                String property_name = property_entity.getSimpleValue().toString();
335                AssociationTypeModel assoc_type_model = new AssociationTypeModel("org.deepamehta.wikidata.assoctype_"
336                        + property_entity.getUri().replaceAll(WD_SEARCH_ENTITIY_DATA_URI_PREFIX, ""),
337                        property_name, "dm4.core.text");
338                association_type = dms.createAssociationType(assoc_type_model, clientState);
339                // 2) Assign to "Wikidata" Workspace
340                assignToWikidataWorkspace(association_type);
341                // 3) Associated search-result-entity to new assoc-type (to keep track)
342                dms.createAssociation(new AssociationModel("dm4.core.association",
343                        new TopicRoleModel(property_entity.getUri(), "dm4.core.default"),
344                        new TopicRoleModel(association_type.getUri(), "dm4.core.default")
345                        ), clientState);
346                log.info("Turned wikidata property \""+ property_entity.getUri() +"\" into DM Association Type!");
347                tx.success();
348            } catch (Error e) {
349                tx.failure();
350                log.warning("OH: The Wikidata Plugin experienced an unforeseen error! "+ e.getMessage());
351            } finally {
352                tx.finish();
353                return association_type;
354            }
355        }
356    
357        @GET
358        @Path("/property/related/claims/{id}")
359        @Produces(MediaType.APPLICATION_JSON)
360        @Override
361        public List<RelatedAssociation> getTopicRelatedAssociations (@PathParam("id") long topicId) {
362            Topic topic = dms.getTopic(topicId, false);
363            List<RelatedAssociation> associations = topic.getRelatedAssociations("dm4.core.aggregation",
364                    "dm4.core.child", "dm4.core.parent", "org.deepamehta.wikidata.claim_edge", false, false);
365            return associations;
366        }
367    
368    
369        // --
370        // ---  Wikidata Search (Application Specific) Private Methods
371        // --
372    
373        private void processWikidataEntitySearch(String json_result, CompositeValueModel search_bucket,
374                String type, String lang) {
375            try {
376                JSONObject response = new JSONObject(json_result);
377                JSONArray result = response.getJSONArray("search");
378                if (result.length() > 0) {
379                    for (int i = 0; i < result.length(); i++) {
380                        JSONObject entity_response = result.getJSONObject(i);
381                        // Check if entity already exists
382                        String id = entity_response.getString("id");
383                        Topic existing_entity = dms.getTopic("uri",
384                                new SimpleValue(WD_SEARCH_ENTITIY_DATA_URI_PREFIX + id), false);
385                        if (existing_entity == null) {
386                            // Create new search entity composite
387                            String name = entity_response.getString("label");
388                            String url = entity_response.getString("url");
389                            //
390                            CompositeValueModel entity_composite = new CompositeValueModel();
391                            entity_composite.put(WD_SEARCH_ENTITY_LABEL_URI, name);
392                            if (entity_response.has("description")) {
393                                String description = entity_response.getString("description");
394                                entity_composite.put(WD_SEARCH_ENTITY_DESCR_URI, description);
395                            }
396                            entity_composite.put(DM_WEBBROWSER_URL, url);
397                            if (entity_response.has("aliases")) {
398                                JSONArray aliases = entity_response.getJSONArray("aliases");
399                                for (int a=0; a < aliases.length(); a++) {
400                                    String alias = aliases.getString(a);
401                                    entity_composite.add(WD_SEARCH_ENTITY_ALIAS_URI,
402                                        new TopicModel(WD_SEARCH_ENTITY_ALIAS_URI, new SimpleValue(alias)));
403                                }
404                            }
405                            // set enity place in resultset
406                            entity_composite.put(WD_SEARCH_ENTITY_ORDINAL_NR, i);
407                            // set entity-type
408                            entity_composite.put(WD_SEARCH_ENTITY_TYPE_URI, type);
409                            // set language-value on entity-result
410                            entity_composite.putRef(WD_LANGUAGE_URI, WD_LANGUAGE_DATA_URI_PREFIX + lang);
411                            TopicModel entity_model = new TopicModel(WD_SEARCH_ENTITIY_DATA_URI_PREFIX + id,
412                                    WD_SEARCH_ENTITY_URI, entity_composite);
413                            // create and reference  entity in wikidata search bucket
414                            search_bucket.add(WD_SEARCH_ENTITY_URI, entity_model);
415                        } else {
416                            // reference existing entity in wikidata search bucket by URI
417                            search_bucket.addRef(WD_SEARCH_ENTITY_URI, WD_SEARCH_ENTITIY_DATA_URI_PREFIX + id);
418                        }
419                    }
420                }
421            } catch (JSONException ex) {
422                log.warning("Wikidata Plugin: JSONException during processing a wikidata entity search response. "
423                        + ex.getMessage());
424            }
425        }
426    
427        private Topic createWikidataSearchEntity(JSONObject entity_response, String lang, ClientState clientState) {
428            Topic entity = null;
429            DeepaMehtaTransaction tx = dms.beginTx();
430            try {
431                String id = entity_response.getString("id");
432                // Create new search entity composite
433                CompositeValueModel entity_composite = buildWikidataEntityModel(entity_response, lang);
434                TopicModel entity_model = new TopicModel(WD_SEARCH_ENTITIY_DATA_URI_PREFIX + id,
435                        WD_SEARCH_ENTITY_URI, entity_composite);
436                entity = dms.createTopic(entity_model, clientState);
437                log.fine("Wikidata Search Entity Created (" +
438                    entity_composite.getString(WD_SEARCH_ENTITY_TYPE_URI)+ "): \"" + entity.getSimpleValue() +"\" - FINE!");
439                tx.success();
440            } catch (Exception ex) {
441                log.warning("FAILED to create a \"Wikidata Search Entity\" caused by " + ex.getMessage());
442                tx.failure();
443            } finally {
444                tx.finish();
445                return entity;
446            }
447        }
448    
449        private Topic updateWikidataEntity(Topic entity, JSONObject entity_response, String lang, ClientState clientState) {
450            DeepaMehtaTransaction tx = dms.beginTx();
451            try {
452                // Update existing search entity topic
453                CompositeValueModel entity_composite = buildWikidataEntityModel(entity_response, lang);
454                TopicModel entity_model = new TopicModel(entity.getId(), entity_composite);
455                dms.updateTopic(entity_model, clientState);
456                log.fine("Wikidata Search Entity Updated (" +
457                    entity_composite.getString(WD_SEARCH_ENTITY_TYPE_URI)+ "): \"" + entity.getSimpleValue() +"\" - FINE!");
458                tx.success();
459            } catch (Exception ex) {
460                log.warning("FAILED to UPDATE \"Wikidata Search Entity\" caused by " + ex.getMessage());
461                tx.failure();
462            } finally {
463                tx.finish();
464                return entity;
465            }
466        }
467    
468        private CompositeValueModel buildWikidataEntityModel(JSONObject entity_response, String lang) {
469            CompositeValueModel entity_composite = new CompositeValueModel();
470            try {
471                String id = entity_response.getString("id");
472                String type = entity_response.getString("type");
473                entity_composite = new CompositeValueModel();
474                // main label
475                if (entity_response.has("labels")) {
476                    JSONObject labels = entity_response.getJSONObject("labels");
477                    JSONObject languaged_label = labels.getJSONObject(lang);
478                    String label = languaged_label.getString("value");
479                    entity_composite.put(WD_SEARCH_ENTITY_LABEL_URI, label);
480                }
481                // main description
482                if (entity_response.has("descriptions")) {
483                    JSONObject descriptions = entity_response.getJSONObject("descriptions");
484                    JSONObject languaged_descr = descriptions.getJSONObject(lang);
485                    String description = languaged_descr.getString("value");
486                    entity_composite.put(WD_SEARCH_ENTITY_DESCR_URI, description);
487                }
488                // aliases
489                if (entity_response.has("aliases")) {
490                    JSONObject aliases = entity_response.getJSONObject("aliases");
491                    JSONArray languaged_aliases = aliases.getJSONArray(lang);
492                    for (int a=0; a < languaged_aliases.length(); a++) {
493                        JSONObject alias_object = languaged_aliases.getJSONObject(a);
494                        String alias = alias_object.getString("value");
495                        entity_composite.add(WD_SEARCH_ENTITY_ALIAS_URI,
496                            new TopicModel(WD_SEARCH_ENTITY_ALIAS_URI, new SimpleValue(alias)));
497                    }
498                }
499                // set wikidata url
500                if (type.equals(WD_SEARCH_ENTITY_TYPE_PROPERTY)) {
501                    entity_composite.put(DM_WEBBROWSER_URL, WIKIDATA_ENTITY_URL_PREFIX
502                            + WIKIDATA_PROPERTY_ENTITY_URL_PREFIX + id);
503                } else {
504                    entity_composite.put(DM_WEBBROWSER_URL, WIKIDATA_ENTITY_URL_PREFIX + id);
505                }
506                // set language-value on entity-result
507                entity_composite.putRef(WD_LANGUAGE_URI, WD_LANGUAGE_DATA_URI_PREFIX + lang);
508                // ### sitelinks
509                /** if (entity_response.has("sitelinks")) {
510                    JSONObject sitelinks = entity_response.getJSONObject("sitelinks");
511                    if (sitelinks.has(lang + "wiki")) {
512                        JSONObject sitelink = sitelinks.getJSONObject(lang + "wiki");
513                        entity_composite.put(DM_WEBBROWSER_URL, sitelink.getString("url"));
514                    } else {
515                        log.warning("There is no sitelink for this item in this language/wiki: " + lang + "wiki");
516                    }
517                } **/
518                entity_composite.put(WD_SEARCH_ENTITY_TYPE_URI, type);
519            } catch (JSONException jex) {
520                log.warning("JSONException during build up of the search-entities composite model");
521                return null;
522            } finally {
523                return entity_composite;
524            }
525        }
526    
527    
528        private void processWikidataClaims(String json_result, Topic wikidataItem, String language_code,
529                ClientState clientState) {
530            try {
531                JSONObject response = new JSONObject(json_result);
532                JSONObject result = response.getJSONObject("claims");
533                // ### Needs to identify if claims (already imported in DM4) are not yet part of the current wikidata-data
534                Iterator properties = result.keys();
535                log.info("Wikidata Plugin is processing all properties part of related CLAIMS");
536                Topic propertyEntity = null;
537                while (properties.hasNext()) {
538                    String property_id = properties.next().toString();
539                    // 1) Load related property-entity
540                    propertyEntity = getOrCreateWikidataEntity(property_id, language_code, clientState);
541                    // HashMap<String, List<Topic>> all_entities = new HashMap<String, List<Topic>>();
542                    JSONArray property_listing = result.getJSONArray(property_id);
543                    for (int i=0; i < property_listing.length(); i++) {
544                        // 2) fetch related wikidata entity
545                        Topic referencedItemEntity = null;
546                        JSONObject entity_response = property_listing.getJSONObject(i);
547                        JSONObject mainsnak = entity_response.getJSONObject("mainsnak");
548                        String claim_guid = entity_response.getString("id");
549                        // 3) build up item as part of the claim (if so)
550                        String itemId = "";
551                        String snakDataType = mainsnak.getString("datatype");
552                        JSONObject snakDataValue = mainsnak.getJSONObject("datavalue");
553                        // ..) depending on the various (claimed/realted) value-types
554                        if (snakDataType.equals("wikibase-item")) {
555                            // log.info("Wikibase Item claimed via \"" + propertyEntity.getSimpleValue() + "\"");
556                            JSONObject snakDataValueValue = snakDataValue.getJSONObject("value");
557                            long numericId = snakDataValueValue.getLong("numeric-id");
558                            itemId = "Q" + numericId; // is this always of entity-type "item"? responses looks like.
559                            referencedItemEntity = getOrCreateWikidataEntity(itemId, language_code, clientState);
560                        } else if (snakDataType.equals("commonsMedia")) {
561                            // do relate wikidata.commons_media
562                            log.fine("Commons Media claimed via \"" + propertyEntity.getSimpleValue()
563                                    + "\" ("+language_code+") DEBUG:");
564                            log.fine("  " + snakDataValue.toString());
565                            // ### make use of WIKIMEDIA_COMMONS_MEDIA_FILE_URL_PREFIX and implement page-renderer
566                        } else if (snakDataType.equals("globe-coordinate")) {
567                            // do relate wikidata.globe_coordinate
568                            log.fine("Globe Coordinate claimed via \"" + propertyEntity.getSimpleValue()
569                                    + "\" ("+language_code+") DEBUG:");
570                            log.fine("  " + snakDataValue.toString());
571                        } else if (snakDataType.equals("string")) {
572                            if (snakDataValue.has("value")) {
573                                String value = snakDataValue.getString("value");
574                                referencedItemEntity = getOrCreateWikidataText(value, language_code, clientState);
575                            } else {
576                                log.warning("Could not access wikidata-text value - json-response EMPTY!");
577                            }
578                        } else {
579                            log.warning("Value claimed as " + propertyEntity.getSimpleValue() + " is not of any known type"
580                                    + " wikibase-item but \"" + snakDataType +"\" ("+snakDataValue+")");
581                            // e.g. snakDataType.equals("quantity")
582                        }
583                        // store topic reference to (new or already existing) wikidata-entity/ resp. -value topic
584                        if (referencedItemEntity != null) {
585                            createWikidataClaimEdge(claim_guid, wikidataItem, referencedItemEntity,
586                                propertyEntity, clientState);
587                        } else {
588                            log.warning("SKIPPED creating claim of type \""+snakDataType+"\" value for "
589                                    + "\""+propertyEntity.getSimpleValue()+"\"");
590                        }
591                    }
592                    /** Iterator entity_iterator = all_entities.keySet().iterator();
593                    StringBuffer requesting_ids = new StringBuffer();
594                    while (entity_iterator.hasNext()) {
595                        String entity_id = entity_iterator.next().toString();
596                        requesting_ids.append(entity_id + "|");
597                    }
598                    log.info("Requesting ALL ITEMS for " +property_id+ ": " + requesting_ids.toString());
599                    omitting this solution bcause: "*": "Too many values supplied for parameter 'ids': the limit is 50" **/
600                }
601            } catch (JSONException ex) {
602                log.warning("JSONException during processing a wikidata claim. " + ex.getMessage());
603            }
604        }
605    
606        private Association createWikidataClaimEdge (String claim_guid, Topic one, Topic two,
607                Topic property, ClientState clientState) {
608            DeepaMehtaTransaction tx = dms.beginTx();
609            Association claim = null;
610            try {
611                if (!associationExists(WD_ENTITY_CLAIM_EDGE, one, two)) {
612                    // 1) Create \"Wikidata Claim\"-Edge with GUID
613                    claim = dms.createAssociation(new AssociationModel(WD_ENTITY_CLAIM_EDGE,
614                        new TopicRoleModel(one.getId(), "dm4.core.default"),
615                        new TopicRoleModel(two.getId(), "dm4.core.default")), clientState);
616                    claim.setUri(claim_guid);
617                    log.info("Created \"Wikidata Claim\" with GUID: " + claim.getUri() +" for \"" + two.getSimpleValue() +
618                                    " (property: " + property.getSimpleValue() +
619                                    "\") for \"" + one.getSimpleValue() + "\" - FINE");
620                    // 2) Assign wikidata property (=Wikidata Search Entity) to this claim-edge
621                    claim.setCompositeValue(new CompositeValueModel().putRef(WD_SEARCH_ENTITY_URI,
622                            property.getUri()), clientState, null);
623                }
624                tx.success();
625            } catch (Exception e) {
626                log.severe("FAILED to create a \"Claim\" between \""+one.getSimpleValue()+"\" - \""+two.getSimpleValue());
627                tx.failure();
628                throw new RuntimeException(e);
629            } finally {
630                tx.finish();
631                return claim;
632            }
633        }
634    
635        private Topic getOrCreateWikidataText(String value, String lang, ClientState clientState) {
636            Topic textValue = null;
637            // 1) query for text-value
638            log.info("Value: " + value);
639            try {
640                textValue = dms.getTopic(WD_TEXT_TYPE_URI, new SimpleValue(value), false);
641            } catch (Exception ex) {
642                log.info("Could not find a wikidata-text value topic for " + value + ex.getMessage());
643            }
644            // 2) re-use  or create
645            DeepaMehtaTransaction tx = dms.beginTx();
646            try {
647                if (textValue == null) {
648                    textValue = dms.createTopic(new TopicModel(WD_TEXT_TYPE_URI, new SimpleValue(value)), clientState);
649                    log.info("CREATED \"Wikidata Text\" - \"" + value +"\" (" + lang + ") - OK!");
650                } else {
651                    log.info("FETCHED \"Wikidata Text\" - \"" + textValue.getSimpleValue() +"\" "
652                            + "(" + lang + ") - Re-using it!");
653                }
654                tx.success();
655            } catch (Exception ex) {
656                log.warning("FAILURE during creating a wikidata value topic: " + ex.getLocalizedMessage());
657                tx.failure();
658            } finally {
659                tx.finish();
660                return textValue;
661            }
662        }
663    
664    
665    
666        // --
667        // --- DeepaMehta 4 Plugin Related Private Methods
668        // --
669    
670        private boolean associationExists(String edge_type, Topic item, Topic user) {
671            List<Association> results = dms.getAssociations(item.getId(), user.getId(), edge_type);
672            return (results.size() > 0) ? true : false;
673        }
674    
675        private void assignToWikidataWorkspace(Topic topic) {
676            // fixme: remove assignment of type to other (selected via clientState) workspace
677            Topic defaultWorkspace = dms.getTopic("uri", new SimpleValue(WS_WIKIDATA_URI), false);
678            dms.createAssociation(new AssociationModel("dm4.core.aggregation",
679                new TopicRoleModel(topic.getId(), "dm4.core.parent"),
680                new TopicRoleModel(defaultWorkspace.getId(), "dm4.core.child")
681            ), null);
682        }
683    
684        /** --- Implementing PluginService Interfaces to consume AccessControlService --- */
685    
686        @Override
687        @ConsumesService({
688            "de.deepamehta.plugins.accesscontrol.service.AccessControlService"
689        })
690        public void serviceArrived(PluginService service) {
691            if (service instanceof AccessControlService) {
692                acService = (AccessControlService) service;
693            }
694        }
695    
696        @Override
697        @ConsumesService({
698            "de.deepamehta.plugins.accesscontrol.service.AccessControlService"
699        })
700        public void serviceGone(PluginService service) {
701            if (service == acService) {
702                acService = null;
703            }
704        }
705    
706    }