5555 'P91' : 'Sexual Orientation' ,
5656 'P69' :'Educated At' ,
5757 'P102' :'Member of political party' ,
58- 'P551' :'Residence'
58+ 'P551' :'Residence' ,
59+ 'P3373' : 'Sibling'
5960}
6061
6162
@@ -71,10 +72,25 @@ def getWDPPropertyLabel(propertyId):
7172 wd2 = request_wd .get (property_url )
7273
7374 if wd2 .status_code == 200 :
74- data2 = wd2 .json ()['entities' ][propertyId ]
75- if 'en' in data2 ['labels' ]:
76- wd_properties [propertyId ]= data2 ['labels' ]['en' ]['value' ]
77- return wd_properties [propertyId ]
75+ log .debug (wd2 .json ())
76+ # data2 = wd2.json()['entities']
77+ for k ,data2 in wd2 .json ()['entities' ].items ():
78+ if 'en' in data2 ['labels' ]:
79+ data2 ['label' ]= data2 ['labels' ]['en' ]['value' ]
80+ if 'mul' in data2 ['labels' ]:
81+ data2 ['label' ]= data2 ['labels' ]['mul' ]['value' ]
82+ if 'label' not in data2 :
83+ if len (data2 ['labels' ]) == 0 :
84+ label = ''
85+ else :
86+ key = next (iter (data2 ['labels' ]))
87+ data2 ['label' ] = data2 ['labels' ][key ]['value' ]
88+
89+ if k != propertyId :
90+ wd_properties [k ]= data2
91+ else :
92+ wd_properties [propertyId ]= data2
93+ return data2
7894 else :
7995 wd_properties [propertyId ]= ''
8096 return wd_properties [propertyId ]
@@ -83,11 +99,13 @@ def getWDPPropertyLabel(propertyId):
8399def processWikidata (performer ,performer_update ,url ):
84100 wikidata_id = url [30 :]
85101 api_url = 'https://www.wikidata.org/wiki/Special:EntityData/%s.json' % (wikidata_id ,)
86- log .debug ('about to fetch wikidata url: %s' % (api_url ,))
102+ log .debug ('about to fetch wikidata url: %s for performer %s ' % (api_url , performer [ 'name' ] ,))
87103 wd = request_wd .get (api_url )
88104 if wd .status_code == 200 :
89105# log.debug(wd.json().keys())
90- data = wd .json ()['entities' ][wikidata_id ]
106+
107+ data2 = wd .json ()['entities' ]
108+ data = data2 [next (iter (data2 ))]
91109 if settings ['wikidatExtraUrls' ]:
92110 urls = []
93111 for claim ,urlstring in wikidata_property_urls .items ():
@@ -110,65 +128,175 @@ def processWikidata(performer,performer_update,url):
110128 performer_update ['urls' ].append (url )
111129 performer_update ['update' ] = True
112130 log .debug (performer_update )
131+
113132 if settings ['awards' ]:
133+ won_award = []
134+ nominated_award = []
135+ award_totals = {}
136+ nominated_totals = {}
114137 # award received (P166)
115138 # nominated for (P1411)
116139 for prop in ['P166' ,'P1411' ]:
117140 if prop in data ['claims' ]:
118141 for c in data ['claims' ][prop ]:
119- # log.debug(c)
142+ log .debug (c )
143+
120144
121145 award = {}
122146 award_id = c ['mainsnak' ]['datavalue' ]['value' ]['id' ]
123147
124- award ['name' ]= getWDPPropertyLabel (award_id )
148+ award ['wd' ]= getWDPPropertyLabel (award_id )
149+ award ['name' ]= award ['wd' ]['label' ]
150+ award ['label' ] = award ['wd' ]['label' ]
151+
152+ if prop == 'P166' :
153+ award ['type' ]= 'award received'
154+ elif prop == 'P1411' :
155+ award ['type' ]= 'nominated'
156+
157+
158+ if 'P1027' in award ['wd' ]['claims' ].keys ():
159+ award ['conferred_wd' ]= getWDPPropertyLabel (award ['wd' ]['claims' ]['P1027' ][0 ]['mainsnak' ]['datavalue' ]['value' ]['id' ])
160+ award ['conferred' ] = award ['conferred_wd' ]['label' ]
161+ if award ['type' ]== 'award received' :
162+ if award ['conferred' ] not in award_totals :
163+ award_totals [award ['conferred' ]]= 0
164+ award_totals [award ['conferred' ]] = award_totals [award ['conferred' ]] + 1
165+ if settings ['createTag' ]:
166+ performer_update ['tag_names' ].append (
167+ '[%s Award Winner]' % (award ['conferred' ],))
168+
169+ else :
170+ if award ['conferred' ] not in nominated_totals :
171+ nominated_totals [award ['conferred' ]]= 0
172+ nominated_totals [award ['conferred' ]] = nominated_totals [award ['conferred' ]] + 1
173+ if settings ['createTag' ]:
174+ performer_update ['tag_names' ].append ('[%s Award Nominated]' % (award ['conferred' ],))
175+ else :
176+ if award ['type' ]== 'award received' :
177+ if 'unknown' not in nominated_totals :
178+ award_totals ['unknown' ] = 0
179+ award_totals ['unknown' ] = award_totals ['unknown' ] + 1
180+ else :
181+ if 'unknown' not in nominated_totals :
182+ nominated_totals ['unknown' ]= 0
183+ nominated_totals ['unknown' ] = nominated_totals ['unknown' ] + 1
184+
185+
186+ # sublcass of, can be award for best scene
187+ if 'P279' in award ['wd' ]['claims' ].keys ():
188+ award ['subclass_wd' ] = getWDPPropertyLabel (award ['wd' ]['claims' ]['P279' ][0 ]['mainsnak' ]['datavalue' ]['value' ]['id' ])
189+ award ['subclass' ] = award ['subclass_wd' ]['label' ]
190+
191+
192+
125193
126194 if 'qualifiers' in c :
127195 for q ,qv in c ['qualifiers' ].items ():
128196 # point in time
129- # log.debug('q=%s qv=%s'% (q,qv,))
197+ log .debug ('q=%s qv=%s' % (q ,qv ,))
130198 if q == 'P585' :
131199 if len (qv )> 0 :
132200 award ['time' ]= qv [0 ]['datavalue' ]['value' ]['time' ][1 :5 ]
133- # Subject of (the event name
201+ # Subject of (the event name)
134202 if q == 'P805' :
135- award ['venue' ] = getWDPPropertyLabel (qv [0 ]['datavalue' ]['value' ]['id' ])
203+ award ['venue_wd' ] = getWDPPropertyLabel (qv [0 ]['datavalue' ]['value' ]['id' ])
204+ award ['venue' ] = award ['venue_wd' ]['label' ]
205+ # Award Rationale
206+ if q == 'P6208' :
207+ award ['name' ]= qv [0 ]['datavalue' ]['value' ]['text' ]
208+
209+
136210
137211 if award :
138- log .info ('award: %s' % (award ,))
212+ # log.info('award: %s' % (award,))
139213 if 'custom_fields' not in performer_update :
140- performer_update ['custom_fields' ]= {'full' :performer ['custom_fields' ]}
214+ performer_update ['custom_fields' ]= {'full' :performer ['custom_fields' ].copy ()}
215+
141216 award_name = award ['name' ]
142- award_value = award ['name' ]
217+ award [ ' award_value' ] = award ['name' ]
143218 if 'venue' in award and 'time' in award :
144- award_value = '%s - %s: %s' % (award ['time' ], award ['venue' ],award ['name' ],)
219+ award [ ' award_value' ] = '%s - %s: %s' % (award ['time' ], award ['venue' ],award ['name' ],)
145220 elif 'time' in award :
146- award_value = '%s: %s' % (award ['time' ],award ['name' ],)
221+ award [ ' award_value' ] = '%s: %s' % (award ['time' ],award ['name' ],)
147222 elif 'venue' in award :
148- award_value = '%s: %s' % (award ['venue' ],award ['name' ],)
149- if prop == 'P1411' :
150- award_value = '%s - Nominated' % award_value
223+ award ['award_value' ]= '%s: %s' % (award ['venue' ],award ['name' ],)
224+
225+
226+ if award ['type' ]== 'award received' :
227+ won_award .append (award )
228+ if award ['type' ]== 'nominated' :
229+ award ['award_value' ]= '%s - Nominated' % award ['award_value' ]
230+ nominated_award .append (award )
151231 if award_name not in performer_update ['custom_fields' ]['full' ]:
152- performer_update ['custom_fields' ]['full' ][award_name ]= award_value
232+
233+ performer_update ['custom_fields' ]['full' ][award_name ]= award ['award_value' ]
153234 performer_update ['update' ] = True
154- log .debug (performer_update )
155- if settings ['createTag' ]:
156- if prop == 'P166' :
157- performer_update ['tag_names' ].append ('[Award Winner]' )
158- performer_update ['update' ] = True
159- elif prop == 'P1411' :
160- performer_update ['tag_names' ].append ('[Award Nominated]' )
161- performer_update ['update' ] = True
235+ else :
236+ if award ['award_value' ] not in performer_update ['custom_fields' ]['full' ][award_name ]:
237+ tmp = performer_update ['custom_fields' ]['full' ][award_name ].split (', ' )
238+ tmp .append (award ['award_value' ])
239+ performer_update ['custom_fields' ]['full' ][award_name ]= ', ' .join (sorted (tmp ,reverse = True ))
240+ performer_update ['update' ] = True
241+ #check what type of
242+ log .debug (award ['wd' ])
243+
244+ log .debug (performer )
245+ if award_totals :
246+ performer_update ['custom_fields' ]['full' ]['award totals' ] = ', ' .join (
247+ [ "%s: %s" % (k ,v ,) for k ,v in award_totals .items ()])
248+ performer_update ['update' ] = True
249+ if nominated_totals :
250+ performer_update ['custom_fields' ]['full' ]['nominated totals' ] = ', ' .join (
251+ ["%s: %s" % (k , v ,) for k , v in nominated_totals .items ()])
252+ performer_update ['update' ] = True
253+ if won_award :
254+ # performer_update['custom_fields']['full']['json_awards'] = json.dumps([x[for x in won_award])
255+ performer_update ['custom_fields' ]['full' ]['Awards Won' ] = ', ' .join (
256+ [x ['award_value' ] for x in won_award ])
257+ if settings ['createTag' ]:
258+ performer_update ['tag_names' ].append ('[Award Winner]' )
259+ performer_update ['update' ] = True
260+ if nominated_award :
261+ # performer_update['custom_fields']['full']['json_nominated'] = json.dumps(nominated_award)
262+ performer_update ['custom_fields' ]['full' ]['Awards Nominated' ] = ', ' .join (
263+ [x ['award_value' ] for x in nominated_award ])
264+ if settings ['createTag' ]:
265+ performer_update ['tag_names' ].append ('[Award Nominated]' )
266+ performer_update ['update' ] = True
267+ # if settings['createTag']:
268+ # if 'P31' in award['wd']['claims']:
269+ # for c in award['wd']['claims']['P31']:
270+ # log.debug('c %s' % (c,))
271+ # # avn Award Q824540
272+ # if c['mainsnak']['datavalue']['value']['id']=='Q824540':
273+ # log.debug('---------------')
274+ # if prop=='P166':
275+ # performer_update['tag_names'].append('[AVN Award Winner]')
276+ # performer_update['update'] = True
277+ # elif prop=='P1411':
278+ # performer_update['tag_names'].append('[AVN Award Nominated]')
279+ # performer_update['update'] = True
280+ #
281+ # if settings['createTag']:
282+ # if prop=='P166':
283+ # performer_update['tag_names'].append('[Award Winner]')
284+ # performer_update['update'] = True
285+ # elif prop=='P1411':
286+ # performer_update['tag_names'].append('[Award Nominated]')
287+ # performer_update['update'] = True
162288 if settings ['otherInfo' ]:
163289 for claim , label in wikidata_field_properties .items ():
164290 if claim in data ['claims' ]:
165291 claim_values = []
166292 for c in data ['claims' ][claim ]:
167293# log.debug(c)
168- claim_values .append (getWDPPropertyLabel (c ['mainsnak' ]['datavalue' ]['value' ]['id' ]))
294+ # some bad data, used th have a property but no longer, maybe it got deleted and this is pointing to the delted item, 'unknown value'
295+ if 'datavalue' in c ['mainsnak' ]:
296+ claim_values .append (getWDPPropertyLabel (c ['mainsnak' ]['datavalue' ]['value' ]['id' ])['label' ])
169297 if len (claim_values )> 0 :
170298 if 'custom_fields' not in performer_update :
171- performer_update ['custom_fields' ] = {'full' : performer ['custom_fields' ]}
299+ performer_update ['custom_fields' ] = {'full' : performer ['custom_fields' ]. copy () }
172300 if label not in performer_update ['custom_fields' ]['full' ]:
173301 performer_update ['update' ] = True
174302 performer_update ['custom_fields' ]['full' ][label ] = ', ' .join (claim_values )
@@ -178,30 +306,38 @@ def processWikidata(performer,performer_update,url):
178306def processPerformer (performer ):
179307
180308 performer_update = {'id' :performer ['id' ],'update' :False ,"tag_names" :[]}
181- # log.debug(performer)
309+ log .debug (performer )
182310 for u in performer ['urls' ]:
183311 if u .startswith ('https://www.wikidata.org' ) and settings ['processWikidata' ]:
184312 processWikidata (performer ,performer_update ,u )
313+
185314 if performer_update ['update' ]:
186- log . debug ( 'needs update' )
315+ needs_update = False
187316 performer_update .pop ('update' )
188317 performer_update ['tag_ids' ]= [x ['id' ] for x in performer ['tags' ]]
189318 for t in performer_update ['tag_names' ]:
190319 tt = stash .find_tag (t , create = True )
191320 if tt ['id' ] not in performer_update ['tag_ids' ]:
192321 performer_update ['tag_ids' ].append (tt ['id' ])
322+ needs_update = True
193323 performer_update .pop ('tag_names' )
194324
195325 if settings ['schema' ] < 71 :
196326 log .info ('your version of stash does not support custom fields, a new version of stash should be released soon' )
197327 # other features will still work for other versions
198328 performer_update .pop ('custom_fields' )
199- log .info ('updating performer: %s' % (performer_update ,))
200- stash .update_performer (performer_update )
329+ else :
330+ if not performer ['custom_fields' ]== performer_update ['custom_fields' ]['full' ]:
331+ needs_update = True
332+ if needs_update :
333+ log .info ('updating performer: %s' % (performer_update ,))
334+ stash .update_performer (performer_update )
335+ else :
336+ log .debug ('no performer update needed' )
201337
202338def processPerformers ():
203339 query = {}
204- count = stash .find_scenes (
340+ count = stash .find_performers (
205341 f = query ,
206342 filter = {"per_page" : 1 },
207343 get_count = True ,
0 commit comments