from bs4 import BeautifulSoup
from bs4 import NavigableString
import urllib.request
import string
import re
import codecs

def CheckLocation(Check_Link_Url,CheckLinkName): #filters the CheckLinkName string, and if passes the filter then scrape the page Check_Link_Url
    FilterList =["",]
    with urllib.request.urlopen(Check_Link_Url) as response:
            data = response.read()

def search_for_overflow(soup):
    
    for  overflow_List in soup.findAll('div', attrs={'style':'overflow-x:auto'}): # we search for the data. In our case the data we were looking for is always 
        #in a div with this specific attribute. these div represented each day of the month. They contained a table that listed all the ships sank in that date. 
        
        date = overflow_List.caption.text
        date = date.replace("List of shipwrecks: ","") # cleans the title of the table for the day, to have the date in a simpler format
        pretty2=overflow_List.prettify() # there must be a more elegant way, but I found simpler to create a new beautifulsoup objec for every div found
        soup2 = BeautifulSoup(pretty2,'lxml')
        for List_tr in soup2.findAll('tr'):# for all the div found we search for the table rows
            pretty3=List_tr.prettify()  # and again we create a soup object of them
            soup3 = BeautifulSoup(pretty3,'lxml')
            for index,List_td in enumerate(soup3.findAll('td')): #then we search for all the cells
                
                stringLinea =re.sub(r'\s+', ' ', List_td.text)# we eliminate all the multiple spaces and transform them in single ones
                stringLinea =re.sub('(\[\d+\])', ' ', stringLinea)#then we delete the quotes like for example "[1]"
                if index<2: # for the first 3 cells in the row
                    stringLinea=stringLinea+"@" # we add @ as character separator
                    stringLinea=stringLinea.replace("\n","")# we delete the endline
                    with codecs.open("List_Results_wikipedia_All_Pages_Plus_Coordinates and Names.txt", "a",encoding='utf8') as file:
                        file.write(stringLinea) #we write the cell on the file
                else: # this is the more complicated fourth cell
                    stringLinea=stringLinea.replace("\n","")
                    m = re.search('(?<=/).+(?=/)', stringLinea) # we search for coordinates in the string
                    if m == None:
                        result = ""
                    else:
                        result = m.group(0) # result = the found coordinates
                        
                    pretty4=List_td.prettify()# once again we create the object for beautifulsoup
                    soup4 = BeautifulSoup(pretty4, 'lxml')
                    stringLinks ="none"
                    if soup4!=None:   
                        ListNameLinks = soup4.findAll('a') #simplifying strategy: wikipedia often puts as a link interesting stuff in the texts, we search for them and store them
                        # to help the subsequent work
                    
                        
                        if len(ListNameLinks)==0: # there are not links in the text
                            stringLinks="none"
                        else:
                            stringLinks=""
                            for index,element in enumerate(ListNameLinks): # put in a single string all the links, separated by the semicolon
                                if index<1: # the case that there is only one link in the text
                                    
                                    stringLinks = element.text
                                    stringLinks = stringLinks.replace("\n","")
                                else: # multiple links
                                    textadd= element.text
                                    textadd= textadd.replace("\n","")
                                    stringLinks = stringLinks+";"+textadd
                    
                    stringLinks =re.sub(r'\s+', ' ', stringLinks)# in some cases the string still had multiple spaces, just for good measure we eliminate them       
                    with codecs.open("List_Results_wikipedia_All_Pages_Plus_Coordinates and Names.txt", "a",encoding='utf8') as file:
                        
                        file.write(stringLinea+"@"+date+"@"+result+"@"+stringLinks+"\n") # we organize the whole string as name@date@coordinates@the list of links in the text.

def main():
    with open("List_Results_wikipedia_All_Pages_Plus_Coordinates and Names.txt", "w"): # this is the file where we will store all the data
        pass
    with open('List of Wikipedia links by month August 1914-December 1918')as fileUrls: # we open the text file where we put all the web pages that we will scrape
        addresses = fileUrls.readlines()
    
    for page in addresses:   # for every page we open it
        with urllib.request.urlopen(page) as response:
            data = response.read()
            print(page)
        soup = BeautifulSoup(data, "lxml") # we create the beautifulsoup object
        search_for_overflow(soup) # and we search for the data that we want with our function
   
if __name__ == "__main__":
    main()