Machine Learning with Wikidata

# -*- coding: utf-8  -*-
import json
import numpy as np
import pandas as pd
import sklearn

from collections import defaultdict
from sklearn import linear_model

# male/female population _must_ not be added unqualified as total population (!)
# this is an error and should be fixed at the item using P1540 and P1539 instead
# (wrong query result may be a manifestation of such)
SELECT ?year (AVG(?pop) AS ?population) ?countryLabel
WHERE
{
  ?country wdt:P31 wd:Q6256;
           p:P1082 ?popStatement .
  ?popStatement ps:P1082 ?pop;
                pq:P585 ?date .
  BIND(STR(YEAR(?date)) AS ?year)

  # IF multiple ?pop values per country per year exist, we prioritize by source
  #       census 1st, others 2nd, estimation(s) 3rd, unknown sources (none supplies P459) last
  # note: wikibase:rank won't help here: each year may have multiple statements for ?pop value
  #       rank:prefered is used for the best value (or values) of the latest or current year
  #       rank:normal may be justified for all of multiple ?pop values for a given year
  OPTIONAL { ?popStatement pq:P459 ?method. }
  OPTIONAL { ?country p:P1082 [ pq:P585 ?d; pq:P459 ?estimate ].
             FILTER(STR(YEAR(?d)) = ?year). FILTER(?estimate = wd:Q791801). }
  OPTIONAL { ?country p:P1082 [ pq:P585 ?e; pq:P459 ?census ].
             FILTER(STR(YEAR(?e)) = ?year). FILTER(?census = wd:Q39825). }
  OPTIONAL { ?country p:P1082 [ pq:P585 ?f; pq:P459 ?other ].
             FILTER(STR(YEAR(?f)) = ?year). FILTER(?other != wd:Q39825 && ?other != wd:Q791801). }
  BIND(COALESCE(
    IF(BOUND(?census), ?census, 1/0),
    IF(BOUND(?other), ?other, 1/0),
    IF(BOUND(?estimate), ?estimate, 1/0) ) AS ?pref_method).
  FILTER(IF(BOUND(?pref_method),?method = ?pref_method,true))
  # .. still need to group if multiple values per country per year exist and
  # - none is qualified with P459
  # - multiple ?estimate or multiple ?census (>1 value from same source)
  # - ?other yields more than one source (>1 values are better than optionally
  #                         supplied estimate, but no census source available)

  SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en" }
  FILTER(?year >= "2005")
}
GROUP BY ?year ?countryLabel
ORDER BY ?year ?countryLabel

YEARS = ["2007", "2008", "2009" ,"2010", "2011", "2012", "2013"] # Years we are interested in


def getList(dict): # To get keys for the dict.
    list = []
    for key in dict.keys():
        list.append(key)

    return list

with open('query.json', 'r') as f: # Downloaded query in a JSON file.
    distros_dict = json.load(f)

allEntries = defaultdict(dict) # saves all the countries in the query with its data

for entry in distros_dict:
    allEntries[entry['countryLabel']].update({entry['year']: entry['population']})

selectedEnt = defaultdict(dict) # saves the countries in the query with its data that has all the values in the YEARS list

for country in allEntries:
    if all(elem in getList(allEntries[country]) for elem in YEARS):
        selectedEnt.update({country: allEntries[country]})

df = pd.DataFrame.from_dict(selectedEnt) # pastes it into pandas
data = pd.DataFrame.transpose(df) # flips the table

.                2007     2008     2009  \
Afghanistan    26349243 27032197 27708187
Algeria        35097043 35591377 36383302
... ... ... ...

data = data[YEARS]

predict = "2013"

X = np.array(data.drop([predict], 1)) # Features
y = np.array(data[predict]) # Labels

x_train, x_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y, test_size = 0.1)

linear = linear_model.LinearRegression()

linear.fit(x_train, y_train)
acc = linear.score(x_test, y_test) # acc = accuracy

print(acc)

print('Coefficient: \n', linear.coef_) # These are each slope value
print('Intercept: \n', linear.intercept_) # This is the intercept

predictions = linear.predict(x_test) # Gets a list of all predictions

print("Country - sklearn guessed value for 2013, the Wikidata values (2007-2012), The Wikidata value (2013)")
for x in range(len(predictions)):
    for country in selectedEnt:
        if x_test[x][0] == selectedEnt[country][YEARS[0]] and x_test[x][1] == selectedEnt[country][YEARS[1]]: # To find the country used in the test data
            print(country, " - ", predictions[x], x_test[x], y_test[x])

0.999650607098148
Coefficient:
[ 0.41969474 -1.01050159 -0.20560013  0.0411049   1.3388236   0.41479332]
Intercept:
36691.20709852874

# -*- coding: utf-8  -*-
import json
import numpy as np
import pandas as pd
import sklearn

from collections import defaultdict
from sklearn import linear_model

YEARS = ["2007", "2008", "2009", "2010", "2011", "2012", "2013"]


def getList(dict):
    list = []
    for key in dict.keys():
        list.append(key)

    return list

with open('query.json', 'r') as f:
    distros_dict = json.load(f)

allEntries = defaultdict(dict)

for entry in distros_dict:
    allEntries[entry['countryLabel']].update({entry['year']: entry['population']})

selectedEnt = defaultdict(dict)

for country in allEntries:
    if all(elem in getList(allEntries[country]) for elem in YEARS):
        selectedEnt.update({country: allEntries[country]})

df = pd.DataFrame.from_dict(selectedEnt)
data = pd.DataFrame.transpose(df)

data = data[YEARS]

predict = "2013"

X = np.array(data.drop([predict], 1)) # Features
y = np.array(data[predict]) # Labels

x_train, x_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y, test_size = 0.1)

linear = linear_model.LinearRegression()

linear.fit(x_train, y_train)
acc = linear.score(x_test, y_test)
print(acc)

print('Coefficient: \n', linear.coef_)
print('Intercept: \n', linear.intercept_)

predictions = linear.predict(x_test)

print("Country - sklearn guessed value for 2013, the Wikidata values (2007-2012), The Wikidata value (2013)")
for x in range(len(predictions)):
    for country in selectedEnt:
        if x_test[x][0] == selectedEnt[country][YEARS[0]] and x_test[x][1] == selectedEnt[country][YEARS[1]]:
            print(country, " - ", predictions[x], x_test[x], y_test[x])