Merge pull request #288 from JohnMulligan/dataviz_reqs_refactor

Groupby endpoint, sources update, blog assets,

Author: derekjkeller

README.md

@@ -110,6 +110,7 @@ local:~/Projects/voyages-api$ docker exec -i voyages-solr solr create_core -c bl
 local:~/Projects/voyages-api$ docker exec -i voyages-solr solr create_core -c sources -d /srv/voyages/solr
 local:~/Projects/voyages-api$ docker exec -i voyages-solr solr create_core -c voyagesources -d /srv/voyages/solr
 local:~/Projects/voyages-api$ docker exec -i voyages-solr solr create_core -c enslaversources -d /srv/voyages/solr
+local:~/Projects/voyages-api$ docker exec -i voyages-solr solr create_core -c enslavedsources -d /srv/voyages/solr
 local:~/Projects/voyages-api$ docker exec -i voyages-api bash -c 'python3 manage.py rebuild_indices'
 ```
 

src/api/blog/serializers.py

@@ -44,9 +44,12 @@ def media_uploads_sub(html):
 		an image in the blog post at the front-end site, naturally, rather than here. we
 		rewrite the html before returning it (until i can find a better system).
 	'''
+	print(html)
 	site_storage_base_url_no_leading_slash=re.sub("^/","",site_storage_base_url)
 	media_uploads_baseurl=f"{OPEN_API_BASE_URL}{site_storage_base_url_no_leading_slash}"
 	clean_html=re.sub(f"(?<=src=\").*?{site_storage_base_url}",media_uploads_baseurl,html)
+	clean_html=re.sub(f"(?<=href=\").*?{site_storage_base_url}",media_uploads_baseurl,clean_html)
+	print(clean_html)
 	return clean_html
 
 class PostSerializer(serializers.ModelSerializer):

src/api/common/management/commands/rebuild_indices.py

@@ -93,6 +93,15 @@ def handle(self, *args, **options):
 					'voyage_source_connections__source__short_ref__name'
 				]
 			},
+			{
+				"model":Enslaved,
+				"core_name":"enslavedsources",
+				"fields":[
+					'id',
+					'enslaved_relations__relation__voyage__voyage_source_connections__source__bib',
+					'enslaved_relations__relation__voyage__voyage_source_connections__source__short_ref__name'
+				]
+			},
 			{
 				"model":EnslaverIdentity,
 				"core_name":"enslaversources",

src/api/common/reqs.py

@@ -258,10 +258,13 @@ def post_req(orig_queryset,s,r,options_dict,auto_prefetch=True,paginate=False):
 			# SPECIAL CASE 2: DOCUMENTARY SOURCES
 			if varName.endswith("__source__ALL"):
 				qsetclassstr=str(orig_queryset[0].__class__)
-				if qsetclassstr=="<class 'voyage.models.Voyage'>":
+				if solrcorenamedict[qsetclassstr]=='voyages':
 					filtered_queryset,results_count=global_search(orig_queryset,searchTerm,core_name='voyagesources')
-				elif qsetclassstr=="<class 'past.models.EnslaverIdentity'>":
+				elif solrcorenamedict[qsetclassstr]=='enslavers':
 					filtered_queryset,results_count=global_search(orig_queryset,searchTerm,core_name='enslaversources')
+				elif solrcorenamedict[qsetclassstr]=='enslaved':
+					filtered_queryset,results_count=global_search(orig_queryset,searchTerm,core_name='enslavedsources')
+				
 				filter_obj.remove(item)
 		# TYPICAL ORM-BASED SEARCH/FILTER
 		for item in filter_obj:

src/api/common/static/Enslaved_options.json

@@ -63,7 +63,7 @@ def _extract_iiif_url(url):
 	def handle(self, *args, **options):
 
 		#NOT RUNNING THIS EXCEPT LOCALLY W/O DJK's APPROVAL
-		exit()
+# 		exit()
 		#screen out sources that lack either pages  
 		sources = Source.objects \
 			.prefetch_related('page_connections') \
@@ -97,10 +97,7 @@ def handle(self, *args, **options):
 				s.has_published_manifest = False
 
 			Source.objects.bulk_update(cleared_sources, ["has_published_manifest"])
-
-# 			sources.update(has_published_manifest=False)
-# 			sources.save()
-		
+			
 		generated_count=0
 
 		for source in sources:
@@ -125,7 +122,7 @@ def handle(self, *args, **options):
 				canvas = []
 				abort = False
 				for i, page in enumerate(pages_with_images, 1):
-# 					print(page.__dict__)
+					print(page.__dict__)
 					iiif_baseimage_url=page.iiif_baseimage_url
 					# A canvas page.
 					host_addr,iiif_suffix = Command._extract_iiif_url(iiif_baseimage_url)

src/api/common/static/Enslaved_options.py

@@ -49,15 +49,7 @@ def handle(self, *args, **options):
 
 		j=json.loads(resp.text)
 
-# 		print(resp,json.dumps(j,indent=2))
-		
 		transkribus_shortrefs=ShortRef.objects.all().exclude(transkribus_docId=None)
-		transkribus_shortrefs=transkribus_shortrefs.exclude(name__icontains="SSC Add Ms")
-# 		transkribus_shortrefs=transkribus_shortrefs.exclude(transkribus_docId__in=[1301052,1594526])
-		
-# 		transkribus_shortrefs=transkribus_shortrefs.filter(transkribus_docId=1296568)
-		
-		# transkribus_shortrefs=transkribus_shortrefs.exclude(transkribus_docId__in=[1301052,1594526])
 
 		print(transkribus_shortrefs)
 
@@ -77,7 +69,8 @@ def handle(self, *args, **options):
 					pd[a]=[b]
 
 		for docId in pd:
-			if docId not in ['25566', '25565', '25564', '25563', '25562', '25561', '25560', '25555', '25552', '25554', '25553', '25551', '25550', '25575', '25557', '25567', '25556', '25510', '25509', '25508', '25506', '25507', '25505', '25504', '25503', '25502', '25501', '25500', '25582', '25576', '25577','1437762','1470747','1470827','1470863','1470876','1487718','1487721','1492986','1493090','1493096','1494915','1494931','1496973','1497235']:
+# 			if docId not in ['25566', '25565', '25564', '25563', '25562', '25561', '25560', '25555', '25552', '25554', '25553', '25551', '25550', '25575', '25557', '25567', '25556', '25510', '25509', '25508', '25506', '25507', '25505', '25504', '25503', '25502', '25501', '25500', '25582', '25576', '25577','1437762','1470747','1470827','1470863','1470876','1487718','1487721','1492986','1493090','1493096','1494915','1494931','1496973','1497235']:
+			if docId not in ['1437762', '1439396', '1438141']:
 				break
 			print('DOCID',docId)
 			pages=pd[docId]
@@ -86,8 +79,8 @@ def handle(self, *args, **options):
 
 			for pagepk in pages:
 				page=Page.objects.get(id=pagepk)
+				print(page)
 				page.transkribus_pageid=None
-				page.transcription=None
 				page.save()
 
 			doc_url="https://transkribus.eu/TrpServer/rest/collections/%s/%s/fulldoc" %(transkribus_collection_id,docId)
@@ -96,67 +89,69 @@ def handle(self, *args, **options):
 
 			c=1
 			for transkribus_page in doc['pageList']['pages']:
-# 				print("PAGES",pages,'COUNT',len(pages))
-# 				print("C=",c)
 				pagetexturls=[]
-				sp=Page.objects.get(id=pages[c-1])
-				print(sp.source_connections.first().source)
 
+				imgFileName=transkribus_page['imgFileName']
 
-				if sp.transcriptions.all() != []:
-					print("transcription already exists")
-				else:
-					imgFileName=transkribus_page['imgFileName']
-					## but as a backstop we can capture the pageid on the transkribus side as well
-					## and then bump them as necessary and re-pull (i don't want to do this!!!!)
-					transkribus_pageId=transkribus_page['pageId']
-					transcripts=transkribus_page['tsList']['transcripts']
-# 					if c==8:
-# 						print(json.dumps(transkribus_page,indent=2))
-					
-					pagetext=''
-					for transcript in transcripts:
-						wordcount=transcript['nrOfCharsInLines']
-						if wordcount>0:
-							transcript_url=transcript['url']
-	# 							print(transcript_url)
-							connection_pool = urllib3.PoolManager()
-							resp = connection_pool.request('GET',transcript_url,headers=auth_headers)
-							tmpfilename="tmp/%s.xml" %docId
-							f = open(tmpfilename, 'wb')
-							f.write(resp.data)
-							f.close()
-							resp.release_conn()
-							d=open(tmpfilename,"r")
-							t=d.read()
-							
-							d.close()
-							pagetree=ET.fromstring(t)
-							textlines_trees=[e for e in pagetree.iter() if e.tag=="{http://schema.primaresearch.org/PAGE/gts/pagecontent/2013-07-15}TextLine"]
-							textlines=[]
-							
-							os.remove(tmpfilename)
-							
-							for textline_tree in textlines_trees:
-								linetext=[e.text for e in textline_tree.iter() if e.tag=='{http://schema.primaresearch.org/PAGE/gts/pagecontent/2013-07-15}Unicode' and e.text is not None]
-								textlines.append(' '.join(linetext))
-							pagetext+='\n'.join(textlines)
-# 					print(pagetext)
-					if re.match('\s+',pagetext,re.S):
-						pagetext=None
-					
-# 					sp.transkribus_pageid=transkribus_pageId
-					Transcription.objects.create(
-						page=sp,
-						language_code='en',
-						text=pagetext,
-						is_translation=False
-					)
-					
-# 					sp.transcriptions.add=pagetext
-# 					sp.save()
-					
-					
+				imgFileName_stripped=re.sub('(\.jpg|\.tif|\.tiff)','',imgFileName)
+				
+				skip=False
+				try:
+					sp=Page.objects.get(image_filename__icontains=imgFileName_stripped)
+					print(sp.source_connections.first().source)
+				except:
+					print(f"IMAGE MATCHING FILENAME DOES NOT EXIST:{imgFileName_stripped}")
+					skip=True
+				if not skip:
+					if sp.transcriptions.all().count() != 0:
+						print(sp.transcriptions.all())
+						print("transcription already exists")
+					else:
+						imgFileName=transkribus_page['imgFileName']
+						
+						## but as a backstop we can capture the pageid on the transkribus side as well
+						## and then bump them as necessary and re-pull (i don't want to do this!!!!)
+						transkribus_pageId=transkribus_page['pageId']
+						transcripts=transkribus_page['tsList']['transcripts']
+						pagetext=''
+						max_ts=0
+						transcript=transcripts[0]
+						
+						if transcript is not None:
+						
+							wordcount=transcript['nrOfCharsInLines']
+							if wordcount>0:
+								transcript_url=transcript['url']
+								connection_pool = urllib3.PoolManager()
+								resp = connection_pool.request('GET',transcript_url,headers=auth_headers)
+								tmpfilename="document/management/commands/data/%s.xml" %docId
+								f = open(tmpfilename, 'wb')
+								f.write(resp.data)
+								f.close()
+								resp.release_conn()
+								d=open(tmpfilename,"r")
+								t=d.read()
+								d.close()
+								pagetree=ET.fromstring(t)
+								textlines_trees=[e for e in pagetree.iter() if e.tag=="{http://schema.primaresearch.org/PAGE/gts/pagecontent/2013-07-15}TextLine"]
+								textlines=[]
+								
+								os.remove(tmpfilename)
+								
+								for textline_tree in textlines_trees:
+									linetext=[e.text for e in textline_tree.iter() if e.tag=='{http://schema.primaresearch.org/PAGE/gts/pagecontent/2013-07-15}Unicode' and e.text is not None]
+									textlines.append(' '.join(linetext))
+								pagetext+='\n'.join(textlines)
+						print(pagetext)
+						if re.match('\s+',pagetext,re.S):
+							pagetext=""
+						
+						Transcription.objects.create(
+							page=sp,
+							language_code='en',
+							text=pagetext,
+							is_translation=False
+						)
 
 
 				c+=1

src/api/document/management/commands/iiif_generate_manifests.py

@@ -32,6 +32,8 @@ class Transcription(models.Model):
 	def __str__(self):
 		if len(self.text)>20:
 			snippet=self.text[:19]+"..."
+		else:
+			snippet=self.text
 		return f"Transcription of page {self.page}: {snippet}"
 
 class Page(models.Model):

src/api/document/management/commands/transkribus_sync.py

@@ -43,24 +43,31 @@ class Meta:
 
 ############ VOYAGES
 
-class PastEnslavedVoyageSerializer(serializers.ModelSerializer):
+
+class PastVoyageOutcomesSerializer(serializers.Serializer):
+	particular_outcome=serializers.CharField()
+	class Meta:
+		fields=['particular_outcome']
+
+class PastEnslavedVoyageSerializer(serializers.Serializer):
 	id=serializers.IntegerField()
 	embarkation=serializers.CharField()
 	disembarkation=serializers.CharField()
 	year=serializers.IntegerField()
 	month=serializers.IntegerField()
 	day=serializers.IntegerField()
 	ship_name=serializers.CharField()
+	outcomes=PastVoyageOutcomesSerializer(many=False)
 	class Meta:
-		model=Voyage
 		fields=[
 			'id',
 			'embarkation',
 			'disembarkation',
 			'year',
 			'month',
 			'day',
-			'ship_name'
+			'ship_name',
+			'outcomes'
 		]
 
 
@@ -106,21 +113,11 @@ def get_gender(self,instance) -> serializers.CharField():
 		else:
 			gender=None
 		return gender
-
 
 	def get_sources(self,instance) -> PastSourceSerializer(many=True):
-		escs=instance.enslaved_source_connections.all()
-		sources_dict={}
-		for esc in escs:
-			page_range=esc.page_range
-			s=esc.source
-			s_id=s.id
-			s.page_ranges=[page_range]
-			if s_id not in sources_dict:
-				sources_dict[s_id]=s
-			else:
-				sources_dict[s_id].page_ranges.append(page_range)
-		return PastSourceSerializer([sources_dict[i] for i in sources_dict],many=True,read_only=True).data
+		source_ids=list(set([i[0] for i in Enslaved.objects.all().filter(id=instance.id).values_list('enslaved_relations__relation__voyage__voyage_source_connections__source')]))
+		sources=Source.objects.all().filter(id__in=source_ids)
+		return PastSourceSerializer(sources,many=True,read_only=True).data
 
 	def get_voyages(self,instance) -> PastEnslavedVoyageSerializer(many=False):
 		#right now, the table layouts, basically everything assume a single voyage per enslaved person
@@ -148,14 +145,24 @@ def get_voyages(self,instance) -> PastEnslavedVoyageSerializer(many=False):
 		if not ship_name:
 			ship_name="ship unknown"
 
+		particular_outcome=v.voyage_outcome.particular_outcome
+		if not particular_outcome:
+			particular_outcome="outcome unknown"
+		else:
+			particular_outcome=particular_outcome.name
+		
+		
 		voyagedict={
 			'id':v.id,
 			'embarkation':embark,
 			'disembarkation':disembark,
 			'year':year,
 			'month':month,
 			'day':day,
-			'ship_name':ship_name
+			'ship_name':ship_name,
+			'outcomes':{
+				'particular_outcome':particular_outcome
+			}
 		}
 
 		return PastEnslavedVoyageSerializer(voyagedict,many=False,read_only=True).data
@@ -233,6 +240,12 @@ def get_names(self,instance) -> ListField(child=serializers.CharField()):
 		return aliases
 
 	def get_sources(self,instance) -> PastSourceSerializer(many=True):
+		source_ids=list(set([i[0] for i in EnslaverIdentity.objects.all().filter(id=instance.id).values_list('aliases__enslaver_relations__relation__voyage__voyage_source_connections__source__id')]))
+		sources=Source.objects.all().filter(id__in=source_ids)
+		return PastSourceSerializer(sources,many=True,read_only=True).data
+
+
+
 		escs=instance.enslaver_source_connections.all()
 		sources_dict={}
 		for esc in escs: