#!/usr/local/bin/python # -*- encoding: iso-8859-15 -*- import requests from requests.adapters import HTTPAdapter import lxml.html import urllib2 from urllib2 import HTTPError , URLError import uuid import os import sys import datetime import re import time from dateutil import parser, tz import textwrap from datetime import timedelta import calendar from dateutil.relativedelta import relativedelta from dateutil.tz import * sys.stdout = open("/var/www/vhosts/thecatchment.org/html/oircal/oircalfinoutffile.txt","w") sys.stderr = open("/var/www/vhosts/thecatchment.org/html/oircal/oircalfinerrffile.txt","w") s = requests.Session() s.mount('http://', HTTPAdapter(max_retries=5)) html = requests.get("http://www.oireachtas.ie/eventsCalendar/conf/en/events/").content dom = lxml.html.fromstring(html) #urls=["http://www.thecatchment.org/oircal/event-38523.ics","http://www.thecatchment.org/oircal/event-38540.ics"] #urls=["http://www.thecatchment.org/oircal/event-30447.ics","http://www.thecatchment.org/oircal/event-27443.ics","http://www.thecatchment.org/oircal/event-27443.ics","http://www.thecatchment.org/oircal/event-19493d.ics"] ##urls = ["http://www.oireachtas.ie/eventsCalendar/conf/en/events/event-27907.ics","http://www.oireachtas.ie/eventsCalendar/conf/en/events/event-25528.ics","http://www.oireachtas.ie/eventsCalendar/conf/en/events/event-26962.ics","http://www.oireachtas.ie/eventsCalendar/conf/en/events/event-25481.ics"] #,"http://www.thecatchment.org/oircal/event-19821c.ics","http://www.thecatchment.org/oircal/event-18327s.ics","http://www.thecatchment.org/oircal/event-18326d.ics", #urls = ["http://www.oireachtas.ie/eventsCalendar/conf/en/events/event-27907.ics","http://www.oireachtas.ie/eventsCalendar/conf/en/events/event-25528.ics","http://www.oireachtas.ie/eventsCalendar/conf/en/events/event-38204.ics","http://www.oireachtas.ie/eventsCalendar/conf/en/events/event-38205.ics","http://www.oireachtas.ie/eventsCalendar/conf/en/events/event-38249.ics","http://www.oireachtas.ie/eventsCalendar/conf/en/events/event-38226.ics"] #"http://www.oireachtas.ie/eventsCalendar/conf/en/events/event-26970.ics"] #"http://www.thecatchment.org/oircal/event-19493d.ics","","http://www.thecatchment.org/oircal/event-21401d.ics","http://www.thecatchment.org/oircal/event-25548d1.ics #"http://www.thecatchment.org/oircal/event-24847s.ics","http://www.thecatchment.org/oircal/event-25452.ics","http://www.thecatchment.org/oircal/event-20404d.ics", #"http://www.thecatchment.org/oircal/event-20408d.ics","http://www.thecatchment.org/oircal/event-19821c.ics","http://www.thecatchment.org/oircal/event-25447d.ics" #"http://www.thecatchment.org/oircal/event-20929c.ics",,"http://www.thecatchment.org/oircal/event-25074s.ics # "http://www.thecatchment.org/oircal/event-20939c.ics","http://www.thecatchment.org/oircal/event-18330c.ics", # "http://www.thecatchment.org/oircal/event-18553s.ics","http://www.thecatchment.org/oircal/event-18553s.ics", # "http://www.thecatchment.org/oircal/event-20926s.ics","http://www.thecatchment.org/oircal/event-20177d.ics"] # "http://www.thecatchment.org/oircal/event-20685d.ics","http://www.thecatchment.org/oircal/event-20170d.ics", # "http://www.thecatchment.org/oircal/event-19820d.ics","http://www.thecatchment.org/oircal/event-20475d.ics", #"http://www.thecatchment.org/oircal/event-18841.ics","http://www.thecatchment.org/oircal/event-18840.ics"] #BEGIN:VCALENDAR #PRODID:-//Microsoft Corporation//Outlook 11.0 MIMEDIR//EN #VERSION:2.0 #METHOD:PUBLISH #BEGIN:VxEVENT #CLASS:PUBLIC #DTSTART:20131023T103000Z #DTEND:20131022T195500Z #LOCATION;LANGUAGE=en-ie:Seanad Chamber, Leinster House #DESCRIPTION;LANGUAGE=en-ie:10.30 a.m.\n\nOrder of Business\n\n2.00 p.m.\n\nStatements on Seanad Reform\n(An Taoiseach)\n\n4.00 p.m.\n\nStatements on the Common Agricultural Policy\n(Minister for Agriculture, Food and the Marine)\n\n5.30 p.m.\n\nPrivate Members Business\n\n7.30 p.m.\n\nMatters on the Adjournment #SUMMARY;LANGUAGE=en-ie:Seanad Éireann #END:VxEVENT #END:VCALENDAR #write files for calendars todaysdatetime = time.strftime('%Y%m%dT%H%M%SZ') top = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//Microsoft Corporation//Outlook 11.0 MIMEDIR//EN\r\nMETHOD:PUBLISH\r\n" #DTSTAMP:" + todaysdatetime + "\r\n" #top descstart = "DESCRIPTION;LANGUAGE=en-ie:" #Dáil Éireann\\n\\n" descriptiontart = "DESCRIPTION;LANGUAGE=en-ie:" descriptiontartmc = "DESCRIPTION;LANGUAGE=en-ie" newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalf.ics', 'w') #committee newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfcomm.ics', 'w') #committee newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfsean.ics', 'w') #seanad newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfdail.ics', 'w') #dail newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalf.ics', 'a').writelines(top) #all #top newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfcomm.ics', 'a').writelines(top) #committee #top newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfsean.ics', 'a').writelines(top) #seanad #top newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfdail.ics', 'a').writelines(top) #dail #top newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfull.ics', 'w') #committee newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfullcomm.ics', 'w') #committee newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfullsean.ics', 'w') #seanad newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfulldail.ics', 'w') #dail newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfull.ics', 'a').writelines(top) #all #top newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfullcomm.ics', 'a').writelines(top) #committee #top newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfullsean.ics', 'a').writelines(top) #seanad #top newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfulldail.ics', 'a').writelines(top) #dail #top start = "BEGIN:VEVENT\r\nCLASS:PUBLIC\r\nTZOFFSETFROM:+0000\r\nTZOFFSETTO:+0100\r\nTZNAME:IST\r\nX-WR-TIMEZONE:Europe/Dublin\r\nDTSTAMP:" + todaysdatetime + "\r\nORGANIZER:mailto:jane_doe@example.com\r\n" #firstwolines #start = "BEGIN:VEVENT\r\nCLASS:PUBLIC\r\nBEGIN:VTIMEZONE\r\nTZID:Europe/Dublin\r\nBEGIN:STANDARD\r\nDTSTART:20161030T010000\r\nTZOFFSETFROM:+0100\r\nTZOFFSETTO:+0000\r\nTZNAME:GMT\r\nEND:STANDARD\r\nBEGIN:STANDARD\r\nDTSTART:20171029T010000\r\nTZOFFSETFROM:+0100\r\nTZOFFSETTO:+0000\r\nTZNAME:GMT\r\nEND:STANDARD\r\nBEGIN:DAYLIGHT\r\nDTSTART:20170326T010000\r\nTZOFFSETFROM:+0000\r\nTZOFFSETTO:+0100\r\nTZNAME:IST\r\nEND:DAYLIGHT\r\nEND:VTIMEZONE\r\nX-WR-TIMEZONE:Europe/Dublin\r\nDTSTAMP:" + todaysdatetime + "\r\nORGANIZER:mailto:jane_doe@example.com\r\n" #start = "BEGIN:VEVENT\r\nCLASS:PUBLIC\r\nORGANIZER:mailto:jane_doe@example.com"+ "\r\nDTSTAMP:" + todaysdatetime + "\r\n" #firstwolines #dtstamp = "DTSTAMP:" + todaysdatetime + "\r\n" #write files for error checking newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurls.txt', 'w') newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'w') newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferror.txt', 'w') newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrorpm.txt', 'w') newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrorhttp.txt', 'w') newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrorurllib.txt', 'w') newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrorpm.txt', 'w') newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalf24error.txt', 'w') newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrorpm2.txt', 'w') newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalf24error2.txt', 'w') #for wrapping description when over 80 according to vcalendar #Topical Issues http://bit.ly/oirwl Feb 19, 2015 04:42PM D??il Chamber, Leinster House #87 #http://bit.ly/oirwl Feb 19, 2015 04:42PM D??il Chamber, Leinster House #72 #http://bit.ly/oirwl D??il Chamber, Leinster House #5`1 #Commencement Matters http://bit.ly/oirwl Feb 19, 2015 10:30AM Seanad Chamber, Leinster House #http://bit.ly/oirwl Feb 19, 2015 10:30AM Seanad Chamber, Leinster House #73 #http://bit.ly/oirwl Seanad Chamber, Leinster House #52 #Transport, Tourism and Sport (Select sub) CR4, LH 2000 2 p.m. (T)\nAGENDA Vehicle Clamping Bill 2014 [Minister for Transport, Tourism and Sp 140 #Feb 19, 2015 04:42PM descplain = "descplainnull" summaryplain = "summaryplainnull" summaryfunc = "summaryfuncnull" descriptionfull = "descriptionfullnull" description = "descriptionnull" previouseventline9 = "null" def split_lines(text): text = text.replace("Department","Dept").replace("Minister","Min").replace("\\nAGENDA","") print "textsl" print text textaf = text[0:145] #print "textaf" #print textaf lines = textaf.split('\n') print "linesbf" print lines for line in lines: line = line.strip() print "linestrip" print line print "lines" print lines regex = re.compile(r'.{1,68}(?:\s+|$)') return '\n '.join(s.rstrip() for line in lines for s in regex.findall(line)) #= #spltln = spltln[0:145] #spltln = spltln.strip() #spltln #textaf = text[0:145] #print "textaf" #line = '\n '.join(s.rstrip() for line in lines for s in regex.findall(line)) def split_linesdontcrop(text): text = text.replace("Department","Dept").replace("Minister","Min").replace("\\nAGENDA","") print "textsldc" print text #textaf = text[0:145] #print "textafdc" #print textaf lines = text.split('\n') print "linesbfsd" print lines for line in lines: line = line.strip() print "linestripsd" print line print "linesdc" print lines regex = re.compile(r'.{1,68}(?:\s+|$)') return '\n '.join(s.rstrip() for line in lines for s in regex.findall(line)) #line = '\n '.join(s.rstrip() for line in lines for s in regex.findall(line)) #return line #DESCRIPTION;LANGUAGE=en-ie: def locurlcatfunction(roomcheck39, roomcheck25): desclistdups = desclists[countfrmstrtplusplus:] for desclistdupb in desclistdupbs: timecheckfornexttime = desclistdup if ( timecheck[3].isdigit() and timecheck[0:4] != "2011" and timecheck[0:4] != "2012" and timecheck[0:4] != "2013" and timecheck[0:4] != "2014" and timecheck[0:4] != "2015" and timecheck[0:4] != "2016"): #if ((not timecheckfornexttime[3].isdigit()) or (timecheckfornexttime [0:4] == "2011") or (timecheckfornexttime [0:4]) == "2012" or (timecheckfornexttime [0:4] != "2013") or (timecheckfornexttime [0:4] == "2014") or (timecheckfornexttime [0:4] == "2015")): #for desclistdup in desclistdups: print "descduplength" print descduplength print "countfrmstrtsubloopb" print countfrmstrtsubloopb #if venue == "Dail" or venue == "Seanad": # # pass #elif venue == "Committee": # countfrmstrtsubloopb = # pass #timecheckfornexttime = desclistdups[countfrmstrtsubloopb].strip() print "timecheckfornexttimecheck1" print timecheckfornexttime print "desclen" print desclen print "countfrmstrtb4plusplus" print countfrmstrtplusplus print "timecheckfornexttime[0:4]2" print timecheckfornexttime[0:4] #hhhhhhhhhhhhhhhhhhhhhhhhhhhh moved endtime if ( timecheckfornexttime[3].isdigit() and timecheckfornexttime[0:4] != "2011" and timecheckfornexttime[0:4] != "2012" and timecheckfornexttime[0:4] != "2013" and timecheckfornexttime[1:4] != "2014" and timecheckfornexttime[1:4] != "5"): print "passedifor1" print "timecheckfornexttime[3].isdigit():" #print "desclistdups[countfrmstrtloop]" #print desclistdups[countfrmstrtsubloopb] desctimeplusplus = timecheckfornexttime print "desctimeplusplus" print desctimeplusplus desctimeplusplus = timecheckfornexttime # time print "desctimeplusplus" # time print desctimeplusplus # time enddesctime = timecheckfornexttime # time print "enddesctime" # time print enddesctime # time #remove any extra "\n"'s enddesctimecheck = enddesctime.split("\\n") print "enddesctimecheck" # time print enddesctimecheck # time enddesctime = enddesctimecheck[0] enddesctime = str(enddesctime) print "enddesctimefin" # time print enddesctime # time print "enddesctime[0:2]" # time print enddesctime[0:2] # time print "enddesctime24" # time print enddesctime # time print "enddesctime[1]" # time print enddesctime[1] # time if ( enddesctime[1].isdigit()): if int(enddesctime[0:2]) > 12: enddesctime24m12 = int(enddesctime[0:2]) - 12 print "enddesctime24m12" # time print enddesctime24m12 # time enddesctime = str(enddesctime24m12) + enddesctime[2:] print "enddesctime" # time print enddesctime # time else: pass else: pass print "enddesctimefixed" # time print enddesctime # time enddesctime = enddesctime.replace("12noon","12.00 pm").replace("12.00noon","12.00 pm").replace("12.00 noon","12.00 pm") enddesctime = enddesctime.replace("12midnight","12.00am").replace("12.00midnight","12.00am") print "enddesctimenoon" # time print enddesctime # time enddesctime = enddesctime.replace("p.m.","pm").replace("a.m.","am").replace("p.m","pm").replace("a.m","am") print "enddesctimepmam2" # time print enddesctime # time #working if "m" in enddesctime: print "m in enddesctime:" try: timeconv2fin = datetime.datetime.strptime(enddesctime,"%I.%M%p").strftime("%H%M%SZ") print timeconv2fin except ValueError, e: #ValueError: print e.args print "url2" print url eargserror = "e.args212" + str(e.args) + "\r\n" urlerror = "url212:" + url + "\r\n" timeerror = "time212:" + enddesctime + "\r\n" newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrorpm2.txt', 'a').writelines(eargserror) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrorpm2.txt', 'a').writelines(urlerror) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrorpm2.txt', 'a').writelines(timeerror) endtime = origendtime pass else: try: timeconv2fin = datetime.datetime.strptime(enddesctime,"%H.%M").strftime("%H%M%SZ") print timeconv2fin except ValueError, e: #ValueError: print e.args print "url224" print url eargserror = "e.args224" + str(e.args) + "\r\n" urlerror = "url224:" + url + "\r\n" timeerror = "time224:" + enddesctime + "\r\n" newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalf24error2.txt', 'a').writelines(eargserror) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalf24error2.txt', 'a').writelines(urlerror) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalf24error2.txt', 'a').writelines(timeerror) endtime = origendtime pass #timeconv2fin = parser.parse(enddesctime) timeconv2fin = str(timeconv2fin) #timeconv2fin = timeconv2fin[11:19] print "timeconvam2[11:19]" # time print timeconv2fin # time #timeconv2fin = parser.parse(timeconv2fin) print "timeconvamparse" # time print timeconv2fin # time #datet = datetime.datetime.strptime(dstartdate,"%Y%m%dT").strftime("%Y%m%dT") #print "datet2" #print datet timez = str(timeconv2fin) print "timez2" print timez timezcheck = timez.replace(":",".") #[11:19] print "timezcheck21" print timezcheck #if timezcheck = "12noon" if timezcheck and not timezcheck.isspace(): print "not timezcheck21" pass else: print "defaulttime2" timezcheck = "120000" #timezcheck = "120000" timez = dstartdate + timezcheck # + "Z" print "timez2endtime" print timez denddatetime = "DTEND;TZID=Europe/Dublin:" + timez + "\r\n" #denddatetime = ":" + timez + "\r\n" dstartdatetime = "DTSTART;TZID=Europe/Dublin:" + timez + "\r\n" #dstartdatetime = ":" + timez + "\r\n" print "denddatetime" print denddatetime else: pass print "noenddatetime" denddatetime = denddatetime countfrmstrtsubloopb = countfrmstrtsubloopb + 1 #countfrmstrtsubloop = countfrmstrtsubloop + 1 else: description = desclistdupb descriptionsean = "desclistdupb" descriptionelsedail = desclistdupb descriptioncomm = desclistdupb summaryb = description #summaryb = description def timefix(timevar): desctime = timevar #change 12noon into 12:00pm desctime = desctime.replace(" ","").replace("12noon","12.00 pm").replace("12.00noon","12.00 pm").replace("12.00 noon","12.00 pm") #.replace(" ","") desctime = desctime.replace("noon","") desctime = desctime.replace(" ","").replace("12midnight","12.00am").replace("12.00midnight","12.00am") desctime = desctime.replace("approx","").strip() desctime = desctime.replace(",",".").strip() desctime = desctime.replace("p.m.","pm").replace("a.m.","am").replace("p.m","pm").replace("a.m","am") desctime = desctime.replace(":",".") print "desctimenoon" #time print desctime #time print "desctime[0:2]" print desctime[0:2] # time print "desctime[1]" # time print desctime[1] # time print "desctimefin" # time print desctime # time if (desctime[1].isdigit()): if int(desctime[0:2]) > 12: desctime24m12 = int(desctime[0:2]) - 12 print "desctime24m12" # time print desctime24m12 # time desctime = str(desctime24m12) + desctime[2:] print "desctime" # time print desctime # time else: pass else: pass print "desctimefixed" # time print desctime # time #regularise the am pm python date modeules don't seem to like "p.m." prefer "pm" desctime = desctime.replace(" ","").replace("p.m.","pm").replace("a.m.","am").replace("p.m","pm").replace("a.m","am") print "desctimepmam" # time print desctime # time print "desctime[0:2]" # time if "m" in desctime: print "m in desctime:" # try to convert to 2,30pm like time to ISO 8601 140000Z time and add it on to the same day date for each event in the multiple event description try: timeconv = datetime.datetime.strptime(desctime,"%I.%M%p").strftime("%H%M%SZ") #error check time conversion except ValueError, e: #ValueError: print e.args print "url112" print url eargserror = "e.args112" + str(e.args) + "\r\n" urlerror = "url112:" + url + "\r\n" timeerror = "time112:" + desctime + "\r\n" #write files for 12 hour error checking newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrorpm.txt', 'a').writelines(eargserror) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrorpm.txt', 'a').writelines(urlerror) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrorpm.txt', 'a').writelines(timeerror) #print i pass #else it already 24 hour so use %H rather then %I else: print "24hr" try: timeconv = datetime.datetime.strptime(desctime,"%H.%M").strftime("%H%M%SZ") except ValueError, e: #ValueError: print e.args print "url124" print url eargserror = "e.args124" + str(e.args) + "\r\n" urlerror = "url124:" + url + "\r\n" timeerror = "time124:" + desctime + "\r\n" #write files for 24 hour time error checking newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalf24error.txt', 'a').writelines(eargserror) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalf24error.txt', 'a').writelines(urlerror) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalf24error.txt', 'a').writelines(timeerror) try: timeconv except NameError, e: #ValueError: print e.args print "url2" print url eargserror = "e.argstimefix1" + str(e.args) + "\r\n" urlerror = "urltimefix1:" + url + "\r\n" timeerror = "timetimefix1" + enddesctime + "\r\n" newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrortimefix1.txt', 'a').writelines(eargserror) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrortimefix1.txt', 'a').writelines(urlerror) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrortimefix1.txt', 'a').writelines(timeerror) pass #else: #print "timeconv was defined." #timeconv = "startdatetime2" #if not timeconv: # timeconv = startdatetime print "timefixb" print timeconv return timeconv def timecodefunction(timecodef): print "timecodef" print timecodef startdatetimefx = timecodef print "startdatetimefx" print startdatetimefx #startdatetimeev = eventlines[6] #DTSTART;TdID=Europe/Dublin:0150310T233000Z #DTSTART;TZID=Europe/Dublin:20150310T170600Z #DTSTART;TZID=Europe/Dublin:20150310T170600Z startdatetimetcf = timecodef[36:43] #startdatetimetcf = timecodef[17:24] startdatetimef = "why" #startdatetimetcf = startdatetimef[36:43] print "startdatetimetcf1" print startdatetimetcf try: stimecodeconv = datetime.datetime.strptime(startdatetimetcf,"%H%M%SZ").strftime("%I.%M%p") #error check time conversion except ValueError, e: #ValueError: print e.args print "urltc" print url eargserror = "e.argstc" + str(e.args) + "\r\n" urlerror = "urltc:" + url + "\r\n" timeerror = "timetc:" + startdatetimetcf + "\r\n" #write files for 12 hour error checking newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrortc.txt', 'a').writelines(eargserror) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrortc.txt', 'a').writelines(urlerror) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrortc.txt', 'a').writelines(timeerror) #print i pass #stimecodeconv = str(stimecodeconv) print "stimecodeconvb4" print stimecodeconv return stimecodeconv def locurlcatfunction(locationtext): #+ "\r\n"\n langlocation = locationtext.split(':') print "langlocation" print langlocation langlocationlength = int(len(langlocation)) if langlocationlength > 1: locationevent = langlocation[1].strip() #+ "\r\n" locationtag = langlocation[0] locationevent = re.escape(locationevent).replace("\ "," ").replace("\r","") location = locationtag + ":" + locationevent + "\r\n" else: location = locationtag + ":" + description + "\r\n" print "location" print location if len(location) < 40: #if location[39]: roomcheck39 = "D" #roomcheck39 = location[39] print "iflocationb" #print location[39] else: #roomcheck = "D" roomcheck39 = location[39] print "elselocationb" print location[39] roomcheck24 = location[24] print "locationtext" + url print locationtext print "roomcheckf139" + url print roomcheck39 print "roomcheck39printed" print "roomcheck24" + url print location[24] #LOCATION;LANGUAGE=en-ie:Committee Room 3, Leinster House 2000 #LOCATION;LANGUAGE=en-ie:Committee Room 3\, Leinster House 2000 if roomcheck24 == "A": #audiovisual committee category = "CATEGORIES:" + "Committee" + "\r\n" catloc = "Committee" #LOCATION;LANGUAGE=en-ie:Seanad Chamber\, Leinster House elif roomcheck39 == "1": category = "CATEGORIES:" + "Comm, CR1" + "\r\n" catloc = "Comm, CR1" elif roomcheck39 == "2": category = "CATEGORIES:" + "Comm, CR2" + "\r\n" catloc = "Comm, CR2" elif roomcheck39 == "3": category = "CATEGORIES:" + "Comm, CR3" + "\r\n" catloc = "Comm, CR3" elif roomcheck39 == "4": category = "CATEGORIES:" + "Comm, CR4" + "\r\n" catloc = "Comm, CR4" elif roomcheck39 == "D": category = "CATEGORIES:" + "Dail" + "\r\n" catloc = "Dail" elif roomcheck24 == "S": category = "CATEGORIES:" + "Seanad" + "\r\n" catloc = "Seanad" else: category = "CATEGORIES:" + "Dail" + "\r\n" catloc = "Dail" #catloc = category.strip() #return location return category, location, catloc # #return def descriptionfunction(descriptiontext, descplain, descriptionfull): #description = eventlines[10] print "descriptiontext" print descriptiontext #global summary #global description #global summarydesc description = descriptiontext.strip() description = re.escape(description).replace("\ "," ").replace("\\\\\\","\\").replace("\\\\","\\") description = description.strip() descsplit = description.split(':') print "descsplith" print descsplit descsplitlength = len(descsplit) #langlocationlength = len(descsplit) descsplitlengthint = int(descsplitlength) descriptiontartmc = "SUMMARY;LANGUAGE=en-ie" resultevent = "" #why if descsplitlengthint > 1: i = 1 while i < descsplitlengthint: resultevent = resultevent + descsplit[i] i += 1 description = resultevent #+ url #+ "\r\n" # + "descfuncif" + "\r\n" else: description = description #+ url #+ "\r\n" # "descfuncelse" + "\r\n" print "descresultfunction" print description print "descplain1" print descplain description = description.replace("–","-") descplain = description descriptionfull = description description = description.strip() #.replace("\r\n ","").replace("\n\\","").replace("\\n", "").replace("\\","") sumdesc = description print "descresulre" print description description = descriptiontartmc + ":" + description # + "\r\n" print "descriptionprestrip" print description description = description.strip() print "descriptionpostsplitbf" print description s = str(description) description = split_lines(s) #####descriptionfull = splitxlinesdontcrop(s) print "descriptionplitl" print description print "descriptionfull" print descriptionfull #description = description.replace("\\\r\\n\\ \\r\\n","") ##description = description.replace("\n","\r\n") ##description = description.replace("\\\r\\","") #description = re.escape(description).replace("\ "," ").replace("\\","") #if #summarydesc = description.replace("DESCRIPTION;LANGUAGE=en-ie:","") #summarydesc = description.replace("\n","\r\n") #description = description + summarydesc + "\r\n" # " + '' + url +']]>' + "\r\n" #####description = decsription.strip() #####s = str(description) #####description = splitxlines(s) print "descriptionplitaf" print description description = description # + "desctest"# + "\r\n" print "descfunc" print description print "descplainfunc" print descplain print "descpfullfunc" print descriptionfull return (description, descplain, descriptionfull) def summaryfunction(summaryfunc, summaryplain): summary = eventlines[10] global description summary = summary.strip() #summary = summarytext.strip() #langsummary = summary.split(':') print "funcsummary" print summary #summaryevent = langsummary[1].strip() #+ "\r\n" #summarytag = langsummary[0] #summaryevent = re.escape(summaryevent).replace("\ "," ").replace("\r","") ##summaryline = eventlines[10] #summary = summarytag + ":" + summaryevent + "\r\n" summary = re.escape(summary).replace("\ "," ").replace("\\\\\\","\\").replace("\\\\","\\") print "summsplitbf" print summary summary = summary.strip() summsplit = summary.split(':') print "summsplit" print summsplit summsplitlength = len(summsplit) #langlocationlength = len(summsplit) summsplitlengthint = int(summsplitlength) summarytartmc = "DESCRIPTION;LANGUAGE=en-ie" resultevent = "" #why if summsplitlengthint > 1: i = 1 while i < summsplitlengthint: resultevent = resultevent + summsplit[i] i += 1 summary = resultevent #+ url #+ "\r\n" # + "summfuncif" + "\r\n" else: summary = summary #+ url #+ "\r\n" # "summfuncelse" + "\r\n" summaryplain = resultevent summary = summarytartmc + ":" + summary #+ "sumtest" # + "\r\n" #oct 2015 change s = str(summary) summary = split_lines(s) #summary = summary + "\r\n" ###s = str(summary) ###summary = splitxlines(s) print "summaryresulta" print summary #summary = summary + "\r\n" #summarydesc = summarydesc.replace("\n","\r\n") #####summaryplain = summaryplain.replace("\\\r\\","") #summarydesc = summarydesc.replace("\n","\r\n") #summary = summary.replace("\\\r\\","") if summary == "DESCRIPTION;LANGUAGE=en-ie:Dáil Schedule": summary = "DESCRIPTION;LANGUAGE=en-ie" + ":"# + description print "scedmatch" elif summary == "D\\\xc3\\\xa1il Schedule": summary = "DESCRIPTION;LANGUAGE=en-ie" + ":"# + description print "scedmatchb" elif summary == "DESCRIPTION;LANGUAGE=en-ie:D\Ã\¡il Schedule": summary = "DESCRIPTION;LANGUAGE=en-ie" + ":"# + description print "scedmatchc" elif summary == "D\Ã\¡il Schedule": print "scedmatchd" summary = description else: print "scedmatchno" if summary == "DESCRIPTION;LANGUAGE=en-ie:Seanad Schedule": summary = "DESCRIPTION;LANGUAGE=en-ie" + ":"# + description print "scedmatchsa" summaryfunc = summary # + "\r\n" # + summarydesc + "\r\n" + "summaryfunc" print "summaryfunc" print summaryfunc print "summaryplain" print summaryplain return (summaryfunc, summaryplain) #return summarydesc def writefunction(nametowrite): #global summary global description global summaryplain global descplain global summaryfunc global descriptionfull newfile = open(nametowrite, 'a').writelines(start) print "descriptionloopb4funca" print description #descriptionline = eventlines[9] #summary, summarydesc = summaryfunction(summaryline, summarydesc) summary, summaryplain = summaryfunction(summaryfunc, summaryplain) print "summarywf" print summary print "summaryplainwf" print summaryplain description, descplain, descriptionfull = descriptionfunction(description, descplain, descriptionfull) #line print "descriptionlinefullloopaffunc" print description print "descplainloopaffunc" print descplain print "descpfullloopaffunc" print descriptionfull #description = splitxlines(s) summaryplusdesc = summary + " " + descriptionfull # + "\r\n" # "\r\n" + print "summaryplusdesc1" print summaryplusdesc s = str(summaryplusdesc) summary = split_linesdontcrop(s) print "summary1d" print summary summary = summary.replace("\n","\r\n") summary = summary.replace("\\\r\\","") #summaryline = "SUMMARY;LANGUAGE=en-ie:" + description #summaryline = summary #summaryline = "SUMMARY;LANGUAGE=en-ie:" + summary ###summaryline = eventlines[10] # + "\r\n" ###summaryline = eventlines[10] summary = summary + "\r\n" #summary = "description" + + description + "\r\n" open(nametowrite, 'a').writelines(summary) #summary = re.escape(summary).replace("\ "," ") #summary = summary + "descplain" + descplain + "\r\n" #thisisreallythedescription #summary = summary.replace("\\\r\\","") locationline = eventlines[8] #+ "\r\n"\n #(urlline, (category, location, catloc) = locurlcatfunction(locationline) print "categoryl" print category #LOCATION;LANGUAGE=en-ie:Dáil Chamber, Leinster House # should add in geolocation line open(nametowrite, 'a').writelines(dstartdatetime) # time open(nametowrite, 'a').writelines(denddatetime) # time open(nametowrite, 'a').writelines(location) #description.append("\r\n") #description = description.replace("\nAGENDA","\r\nAGENDA") #description = line #description = description.rstrip() #description = description + "\r\n" # + "reallysummary" print "descriptiontring1" print description #description = re.escape(description).replace("\ "," ") description = description.replace("\n","\r\n") description = description.replace("\\\r\\","") description = description + "\r\n" #description = description + summarydesc + "\r\n" open(nametowrite, 'a').writelines(description) #eventlines[4:-2]) open(nametowrite, 'a').writelines(category) #print newfile locationline = eventlines[8] print "roomlocf1" + url print location print "categoryl" print category #urlline = "URL:" + "http://bit.ly/oir-tv" + "\r\n" urlline = "URL:" + url + "\r\n" newfile = open(nametowrite, 'a').writelines(urlline) ocuuid = str(uuid.uuid4()) print ocuuid ocuuidline = "UID:" + ocuuid + urlnum + "\r\n" newfile = open(nametowrite, 'a').writelines(ocuuidline) open(nametowrite, 'a').writelines(eventlines[11:-1]) #endline = "END:VxEVENT" + "\r\n" #open(nametowrite, 'a').writelines(endline) def writefullunction(nametowrite): #global summary global description global summaryplain global descplain global summaryfunc global descriptionfull newfile = open(nametowrite, 'a').writelines(start) print "descriptionfullloopb4funca" print description #descriptionline = eventlines[9] #summary, summarydesc = summaryfunction(summaryline, summarydesc) summary, summaryplain = summaryfunction(summaryfunc, summaryplain) print "summary" print summary print "summaryplain" print summaryplain description, descplain, descriptionfull = descriptionfunction(description, descplain, descriptionfull) #line print "descriptionlinefullloopaffunc" print description print "descplainlinefullloopaffunc" print descplain #description = splitxlines(s) summaryplusdesc = summary + descplain + "\r\n" # "\r\n" + print "summaryplusdesc" print summaryplusdesc s = str(summaryplusdesc) summary = split_linesdontcrop(s) summary = summary + "\r\n" print "summaryplusdescf" print summary print "summaryplusdesc" print summary #summary = summary + "descplain" + descplain + "\r\n" #thisisreallythedescription locationline = eventlines[8] #+ "\r\n"\n #(urlline, (category, location, catloc) = locurlcatfunction(locationline) print "categoryl" print category #LOCATION;LANGUAGE=en-ie:Dáil Chamber, Leinster House # should add in geolocation line open(nametowrite, 'a').writelines(dstartdatetime) # time open(nametowrite, 'a').writelines(denddatetime) # time open(nametowrite, 'a').writelines(location) #description.append("\r\n") #description = descriptionfull description = description.replace("\nAGENDA","\r\nAGENDA") #description = line #description = description.rstrip() description = description + "\r\n" # + "reallysummary" print "descriptiontring" print description #description = description + summarydesc + "\r\n" open(nametowrite, 'a').writelines(description) #eventlines[4:-2]) open(nametowrite, 'a').writelines(category) #print newfile ##summary = summary.replace("\n","\r\n") ##summary = summary.replace("\\\r\\","") #summaryline = "SUMMARY;LANGUAGE=en-ie:" + description #summaryline = summary #summaryline = "SUMMARY;LANGUAGE=en-ie:" + summary ###summaryline = eventlines[10] # + "\r\n" ###summaryline = eventlines[10] #summary = "description" + + description + "\r\n" open(nametowrite, 'a').writelines(summary) locationline = eventlines[8] print "roomlocf1" + url print location print "categoryl" print category #urlline = "URL:" + "http://bit.ly/oir-tv" + "\r\n" urlline = "URL:" + url + "\r\n" newfile = open(nametowrite, 'a').writelines(urlline) ocuuid = str(uuid.uuid4()) print ocuuid ocuuidline = "UID:" + ocuuid + urlnum + "\r\n" newfile = open(nametowrite, 'a').writelines(ocuuidline) open(nametowrite, 'a').writelines(eventlines[11:-1]) #endline = "END:VxEVENT" + "\r\n" #open(nametowrite, 'a').writelines(endline) def urlfunction(commdesctext): print "url" def committeefunction(commdesctext): global description global summary #reading off http://www.oireachtas.ie/eventsCalendar/conf/en/events/ loop for title venue #uncomment ##when reading from oireachtas for entry in dom.cssselect('tbody'): #global summary #global description title = entry.cssselect('.odd')[0].text_content(); #, title = title.strip() venue, rest = entry.cssselect('.odd')[2].text_content().split(' ', 1); #, url = "http://www.oireachtas.ie/eventsCalendar/conf/en/events/" + entry.cssselect('a')[1].get('href'); #, #, time.sleep(0.1) # read each #for when using list local urls #for url in urls: #time.sleep(0.1) urlnum = url.replace("http://www.thecatchment.org/oircal/","").replace(".ics","").replace("http://www.oireachtas.ie/eventsCalendar/conf/en/events/","") urlnewline = url + "\r\n" newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurls.txt', 'a').writelines(urlnewline) print "urlb4try" print url try: eventlines = urllib2.urlopen(url).readlines() #.decode('utf-8') #errorcheck url http error except urllib2.HTTPError, e: print e.code print url #write http error vars httpeargserror = "e.args2http" + str(e.args) + "\r\n" httperror = "url2:" + url + "\r\n" #writehttperror error checking files newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrorhttp.txt', 'a').writelines(httpeargserror) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrorhttp.txt', 'a').writelines(httperror) pass #errorcheck url url error except urllib2.URLError, e: #ValueError: print e.args print url #writeurlerror vars urllibeargserror = "e.args2urllib" + str(e.args) + "\r\n" urlliberror = "url2:" + url + "\r\n" #write url error error checking files newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrorurlib.txt', 'a').writelines(urllibeargserror) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalferrorurlib.txt', 'a').writelines(urlliberror) print "eventlines" print eventlines #get location = venue to split into seperate committee/seanad/dail files print "previouseventline9" print previouseventline9 if eventlines[9] != previouseventline9: #break; print "notdupduplicate" locationline = eventlines[8] #+ "\r\n" #LOCATION;LANGUAGE=en-ie:Committee Room 1, Leinster House 2000 #find first word after ":" langlocation = locationline.split(':') print "langlocation" print langlocation shortlocation = langlocation[1].split(' ') print "shortlocation" print shortlocation venue = shortlocation[0] locationstart = langlocation[0].split(' ') print "locationstart" print locationstart venue = venue.strip() print "venue" print venue if venue == "Audio": venue = "Committee" print venue print url elif venue == "Committee": venue = "Committee" print venue print url elif venue == "Seanad": venue = "Seanad" print venue print url else: venue = "Dail" print url #leave dail to all else to aovid issues with special character in dail Dáil may receive committee and seanad events if they have erros in them #print venue #split each event into a description by \n\n escaped DESCRIPTION;LANGUAGE=en-ie:Dáil Éireann\n\n9.30 a.m.\n\nQuestions (Tanaiste and Minister for Foreign Affairs and Trade)\n\n10.45 a.m. \n\nFines (Payment and Recovery) Bill 2013 - Order for Report, Report and Final Stages\n(Department of the Environment, Community and Local Government)\n\nHealth Service Executive (Financial Matters) Bill 2013 - Second Stage (resumed)\n(Department of Health)\n\n12 noon\n\nLeaders Questions\n\n12.21 p.m.\n\nOrder of Business\n \n12.51 p.m.\n\nTopical Issues\n\nSOS (1 Hour)\n\nFines (Payment and Recovery) Bill 2013 - Order for Report, Report and Final Stages\n(Department of the Environment, Community and Local Government)\n\nHealth Service Executive (Financial Matters) Bill 2013 - Second Stage (resumed)\n(Department of Health)\n\n5.00 p.m.\n\nStatements on Recent Flooding (to conclude at 7.30 pm if not previously concluded)\n(Department of the Environment, Community and Local Government)\n \n7.30 p.m.\n\nPrivate Members Business - Motion re Garda Síochána Ombudsman Commission (Sinn Féin)\n\n9.00 p.m.\n\nDáil Adjourns #chop off the new line at the end #strip DESCRIPTION;LANGUAGE=en-ie: to re-suffix later desc = eventlines[9] #.strip("DESCRIPTION;LANGUAGE=en-ie:") #.strip("DESCRIPTION;LANGUAGE=en-ie:Dáil Éireann") print "desc1" desc = desc.replace('DESCRIPTION;LANGUAGE=en-ie:','') print desc desc = ' '.join(desc.split()) print "desc2postjoin" print desc #deal with any extra spaces between the "\n \n" 's desc = desc.replace('\\nAGENDA\:','AGENDA') print "descagendaclear1" desc = desc.replace('\nAGENDA\:','AGENDA') print "descagendaclear2" desc = desc.replace('\nAGENDA:','AGENDA') print "descagendaclear3" desc = desc.replace('AGENDA:',' AGENDA') print "descagendaclear4" print "clear agenda" desc = filter(None, desc) desc = desc.replace('m:','m\\n\\n') desc = desc.replace('\\n \\n','\\n\\n')#.replace(':','.') #.split("\\n \\n") committee session change desc = desc.replace('p.m.\\n','p.m.\\n\\n').replace('a.m.\\n','a.m.\\n\\n').replace('p.m. \\n','p.m.\\n\\n').replace('a.m \\n','a.m.\\n\\n').replace('p.m\\n','p.m.\\n\\n') desc = desc.replace(' am.',' a.m.') desc = desc.replace('. a.m.',' a.m.') desc = desc.replace('. p.m.',' p.m.') desc = desc.replace('pm.','p.m.') desc = desc.replace('pm:','pm') desc = desc.replace('am:','am') desc = desc.replace('ireann\\n','ireann\\n\\n') desc = desc.replace('a,m.','a.m.') desc = desc.replace('p,m.','p.m.') desc = desc.replace('a,m ','a.m.') desc = desc.replace('.m ','.m.\\n\\n') desc = desc.replace('p,m ','p.m.\\n\\n') desc = desc.replace('a.m ','a.m.\\n\\n') desc = desc.replace('p.m ','p.m.\\n\\n') desc = desc.replace('a.m. ','a.m.\\n\\n') desc = desc.replace('p.m. ','p.m.\\n\\n') desc = desc.replace('m:','m.\\n\\n') desc = desc.replace('.m:','.m.\\n\\n') desc = desc.replace('pm\\n','p.m.\\n\\n') desc = desc.replace('am\\n','a.m.\\n\\n') desc = desc.replace('\\n\\n\\n','\\n\\n') #desc = desc.replace('12 noon','12.00 p.m.\\n\\n') desc = desc.replace('12.00 noon','12.00 p.m.\\n\\n') print "desclist1a" print desc desc = desc.replace('\\n\\n\\n','\\n\\n') #ducation and Social Protection (Select) CR3, LH 2000 3 p.m.\nAGENDA: Private Meeting #desclist1a #ducation and Social Protection (Select) CR3, LH 2000 3 p.m.\n\nAGENDA Private Meeting print "desclist1b" print desc desc = desc.replace('12 noon','12.00 p.m.').replace("12.00 noon","12.00 p.m.").replace(".m.",".m.\\n\\n").replace("\\n\\n\\n\\n","\\n\\n") desc = desc.replace('12 midnight','12.00 a.m.') desc = desc.replace('Session A:','\\n\\n').replace('Session B:','\\n\\n').replace('Session C:','\\n\\n').replace('Session D:','\\n\\n').replace('Session E:','\\n\\n').replace('Session F:','\\n\\n').replace(' - Vote','\\n\\n') #desc = desc.replace('.m. ','.m.\\n\\n') #print "desclist1c" #print desc #desc = desc.replace('.m ','.m.\\n\\n') print "desclist1d" print desc desclists = desc.split("\\n\\n") #fix 2016 format change desclistdups = desc.replace('\\n \\n','\\n\\n').split("\\n\\n") #.split("\\n \\n") print "desclist2" print desclists descliststart = 0 for desclist in desclists: print "desclisy" print desclist desclists[descliststart] = desclist.replace('\\n(',' (').replace('\\n','\\n\\n') descliststart += 1 i = 0; for desclist in desclists: #desclist[i] = desclist[i].replace("\\n","").replace("\\","") if len(desclists[i]) == 0: desclists[i] #? print "desclistclean0" print desclist i += 1; i = 0; for desclistdup in desclistdups: #desclist[i] = desclist[i].replace("\\n","").replace("\\","") if len(desclistdups[i]) == 0: desclistdups[i] print "desclistcleandups" print desclist i += 1; print "desclisyn" print desclists print "desclists\(" print "desclists\(" print desclists #get length of description list to see how many times to loop to get each event description startdatetime = eventlines[6] #dend enddatetime = eventlines[7] #add ;TZID=Europe/Dublin: into #DTSTART > here < 20140326T093000Z #DTEND > here < 20140326T093000Z startdatetimelist = startdatetime.split(':') print "startdatetimelist" print startdatetimelist startdatetimefull = startdatetimelist[0] + ";TZID=Europe/Dublin:" + startdatetimelist[1] #first/ #startdatetimefull = startdatetimelist[0] + ":" + startdatetimelist[1] #first/ dstartdatetime = startdatetimelist[0] + ";TZID=Europe/Dublin:" + startdatetimelist[1] #dstartdatetime = startdatetimelist[0] + ":" + startdatetimelist[1] dendstartdatetime = startdatetimelist[0] + ";TZID=Europe/Dublin:" + startdatetimelist[1] #dendstartdatetime = startdatetimelist[0] + ":" + startdatetimelist[1] startdatetimefull = startdatetimelist[0] + ";TZID=Europe/Dublin:" + startdatetimelist[1] enddatetime = eventlines[7] origenddatetime = eventlines[7] dorigenddatetime = origenddatetime.split(':') enddatetimelist = enddatetime.split(':') print "enddatetimelist" print enddatetimelist denddatetime = enddatetimelist[0] + ";TZID=Europe/Dublin:" + enddatetimelist[1] #denddatetime = enddatetimelist[0] + ":" + enddatetimelist[1] dorigenddatetime = dorigenddatetime[0] + ";TZID=Europe/Dublin:" + dorigenddatetime[1] #dorigenddatetime = dorigenddatetime[0] + ":" + dorigenddatetime[1] #dendstartdatetime = enddatetimelist[0] + ";TZID=Europe/Dublin:" + enddatetimelist[1] #set variables check to see if not changing, see if looping properly summary = "sumdidntchange" desctime = "desctimenull" enddesctime = "enddesctimenull" descdesc = "descdesctimenull" # #DTSTART:20140306T093000Z #DTEND:20140306T113000Z dendtimeev = eventlines[6] dendtime = dendtimeev[9:24] endstarttime = dendtimeev[17:24] print " endstarttime" print endstarttime print "dendtime" print dendtime #DTSTART:20140212T093000Z #DTEND:20140212T210000Z #select datetime part remove dstartpart denddate = dendtimeev[8:17] print "denddate" print denddate #DTEND;TZID=Europe/Dublin:20131023T160000Z #DTEND:20131008T190000Z # # # #DTSTART:20140306T093000Z #select datetime part remove dstartpart startdatetimeev = eventlines[6] print "startdatetimeev1" print startdatetimeev dstarttime = startdatetimeev[9:24] #dendstartdatetime = startdatetimeev[17:24] startdate = startdatetimeev[8:16] print "dendstartdatetime1a" print dendstartdatetime print "startdate1" print startdate print "dstarttime1" print dstarttime #startdatetime = startdatetimeev[17:23] print "startdatetime1b" print startdatetime #DTEND;TZID=Europe/Dublin:20131023T160000Z #DTEND:20131008T190000Z nowdate = datetime.datetime.now().date() #2016-01-19 #nowdate = today.strftime("%Y/%m/%d") #u = i.strftime('%Y/%m/%d') ##u = %s/%s/%s % (i.day, i.month, i.year) print "nowdate" print nowdate dmonth = 1 ##d = datetime.timedelta(days=21) #datetime. ##t = u + d ##print t ###def add_two_weeks(dt0): ## #dt1 = dt0.replace(days=1) ### dt2 = dt0 + timedelta(days=14) ## #dt3 = dt2.replace(days=1) ### return dt2 #startdatem2w = nowdate - timedelta(days=14) startdatem2w = nowdate - relativedelta(months=1) #startdatep2w = nowdate + timedelta(days=14) startdatep2w = nowdate + relativedelta(months=1) print "startdatem2w" print startdatem2w print "startdatep2w" print startdatep2w dstarttimeev = eventlines[6] #select date part of datetime TZ ddstartdate = dstarttimeev[8:16] print "ddstartdate" print ddstartdate #dstartdateslash = dstartdate[0:4] + "/" + dstartdate[4:6] + "/" + dstartdate[6:8] dyear = int(ddstartdate[0:4]) print "dyear" print dyear dmonth = int(ddstartdate[4:6]) print "dmonth" print dmonth dday = int(ddstartdate[6:8]) print "dday" print dday #dstartdateslash = datetime.date(dstartdate[0:4],dstartdate[4:6],dstartdate[6:8]) ddstartdateslash = datetime.datetime(dyear,dmonth,dday).date() #datetime.date(2010, 5, 24) #http://stackoverflow.com/questions/15134424/datetime-datetime-day-field-out-of-range #d = datetime.date(targetYear,targetMonth,1) #d = d + datetime.timedelta(days=targetDayOfMonth-1) print "ddstartdateslash" print ddstartdateslash #select date part of datetime TZ dstartdate = startdatetimeev[8:17] print "desclistsstart" print desclists desclists = filter(None, desclists) endstartdate = startdatetimeev[8:17] print "endstartdate1" print endstartdate desclen = len(desclists) print "desclen1" print desclen desclength = int(desclen) print "desclength1" print desclength print desclists if desclength > 1: if venue == "Dail" or venue == "Seanad": # del desclists[0] del desclistdups[0] elif venue == "Committee": pass else: pass #dend print "descliststartafterdel" print desclists desclen = len(desclists) print "desclenx" print desclen desclength = int(desclen) print "desclengthx" print desclength #dstart print "dstartdate" print dstartdate i = 0; print "desclists" print desclists for desclist in desclists: print "desclength" print desclength print "url" print url #desclist[i] = desclist[i].replace("\\n","").replace("\\","") desclists[i] = desclists[i].replace("\\n","").replace("\\","") desclists[i] = desclists[i].replace("(approx)","") desclists[i] = desclists[i].replace("approx","") desclists[i] = desclists[i].strip() print "desclistclean" print desclist i += 1; print "desclistfn" print desclists description = "didn'tworkfordesc" #if venue == "Dail" or venue == "Seanad": #or venue == "Committee": #if desc description is more then one event then loop other wise pass to replicate it straight into file print "ddstartdate" print ddstartdate j = 0; #check = "check" #if check == "check": print "ddstartdateslash" print ddstartdateslash print "startdatep2w" print startdatep2w print "startdatem2w" print startdatem2w if ddstartdateslash > startdatem2w and ddstartdateslash < startdatep2w: print "within two weeks" print "ddstartdateslash" print ddstartdateslash #if check == "check": if desclength > 1: print "desclistsdesclength" print desclists print "desclengthfdd" print desclength #looking for time, description, end time in declist count , +1, +2 countfrmstrt = 0 print "desclists" print desclists i = 0; timecheck = "timechecknull" #if venue == "Committ ee": for desclist in desclists: print "desclengthfdd" print desclength print "countfrmstrtfor" print countfrmstrt ########description is the key list that im creating and adding to each seperate event description########## #description = ["DESCRIPTION;LANGUAGE=en-ie:" + "\\n\\n"] description = "fordidntwork" #["DESCRIPTION;ENCODING=QUOTED-PRINTABLE:" + "\\n\\n"] #startdatetime = startdatetimefull #"DTSTART;TZID=Europe/start/Dublin:" #dstarttime = startdatetime #"DTSTART;TZID=Europe/Dublin:" + timez + "\r\n" print "endstartdate1b" print endstartdate print "endstartdate1b" print endstarttime #dendstartdatetime = "DTSTART;TZID=Europe/Dublin:" + endstartdate + endstarttime + "\r\n" #dstartdatetime = "DTSTART;TZID=Europe/Dublin:" + endstartdate + endstarttime + "\r\n" #startdatetime = "DTSTART;TZID=Europe/Dublin:" + endstartdate + endstarttime + "\r\n" #endstartdatetime = "DTSTART;TZID=Europe/Dublin:" + endstartdate + endstarttime + "\r\n" #ddstartdatetime = dendstartdatetime #+ "\r\n" # # print "dendstartdatetime1c" #print dendstartdatetime #startdatetime = dendstartdatetime #timez = dstartdate + timezconverted print "dstartdatetime1d" print dstartdatetime summary = "didn'tworksum" print "descriptiontart" print description print "desclistfirst" print desclist i = 0; #countfromstart is the time index, plus 1 is its description and plusplus is the start time of the next event therefore probably the time end of of this event countfrmstrtplus = countfrmstrt + 1 countfrmstrtplusplus = countfrmstrt + 2 #put first item of list into timecheck print "timecheckdesclists[length]" print "countfrmstrt1" print countfrmstrt print "countfrmstrtplus" print countfrmstrtplus print "countfrmstrtplusplus" print countfrmstrtplusplus #if the count has gone through the desclist list if countfrmstrt < desclength: #resultcount timecheck = desclists[countfrmstrt]; print "timecheck" print timecheck print "timecheck[0]" print timecheck[0] print "countfrmstrtislessorequaldesclength:" summary = "didn'tworksum" description = "didntwork" print "timecheck4" print timecheck[0:4] # check if digit, check if is time and check if not year as some event description start with a year #if venue == "Dail" or venue == "Seanad": #or print "passedifor1" print "isdigit" #get event from event descrition and clean print "desclists[countfrmstrt]check" print desclists[countfrmstrt] descevent = desclists[countfrmstrt].strip() print "descevent" print descevent description = descevent #summary = description # get the description which will be the next item on in the list from the time item #and put it in summary rather then just Dáil Schedule # does this work for seanad and committees too? #maybe not for committee as longer description ##do an if seanad if dail and leave summary as original for committees? ###if countfrmstrtplus isn't there? \n\nDáil Adjourns \n\n event-25529.ics print venue #if venue == "Dail" or venue == "Seanad": # print "countfrmstrt" print countfrmstrt #summary += "\r\n" print "len(timecheckfornexttime)" print len(timecheck) print timecheck[0:4] print timecheck[0:4] if len(timecheck) > 5: #why? print "timecheck" print timecheck print "timecheck[3]" print timecheck[3] if ( timecheck[3].isdigit() and timecheck[0:4] != "2011" and timecheck[0:4] != "2012" and timecheck[0:4] != "2013" and timecheck[0:4] != "2014" and timecheck[0:4] != "2015" and timecheck[0:4] != "2016"): desctime = timecheck # time print "desctimeiftimecheck" # time print desctime # time print "countfrmstrtplus" print countfrmstrtplus #if countfrmstrtplus isn't there? \n\nDáil Adjourns \n\n event-25529.ics if countfrmstrtplus > desclength: #isn't there? \n\nDáil Adjourns \n\n event-25529.ics print "desclists[countfrmstrtplus:]" print desclists[countfrmstrtplus:] description = desclists[countfrmstrtplus]; #error else: description = "Adjourns" #Adjourns #Dáil Adjourns summary = eventlines[10] #summary = description #i think for when they extra stray \n not sure nessecary #desctimecheck = desctime.split(" ") #desctimecheck = desctimecheck[0].split("\\n") desctimecheck = desctime.split("\\n") print "desctimecheck1" # time print desctimecheck # time desctime = desctimecheck[0] # check if there is am or pm in the time if not, time is probably already in 24 hour clock #data inconsistency desctime = desctime.replace(":",".") timeconv = timefix(desctime) print "url1" print url #convert time to string timeconv = str(timeconv) print "timeconvam[11:19]" # time print timeconv # time #python time parse still wants 12.00pm to be noon so change it back timeconv = timeconv.replace(" ","").replace("12noon","12.00 pm").replace("12.00 noon","12.00 pm") timeconv = timeconv.replace(" ","").replace('12 midnight','12.00am') print "timeconvamnoon1" # time print timeconv # time #don't need to do this #datet = datetime.datetime.strptime(dstartdate,"%Y%m%dT").strftime("%Y%m%dT") #print "datet" # datet #timez = "120000" timez1 = str(timeconv) print "timez1" print timez1 print "startdate3" print startdate dddstartdatetime = "DTSTART;TZID=Europe/Dublin:" + startdate + "T" + timez1 + "\r\n" #dddstartdatetime = "DTSTART:" + startdate + "T" + timez1 + "\r\n" ddddstartdatetime = "DTSTART;TZID=Europe/Dublin:" + startdate + "T" + timez1 + "\r\n" #ddddstartdatetime = "DTSTART:" + startdate + "T" + timez1 + "\r\n" print "dddstartdatetime3" print dddstartdatetime #print "dddstartdatetime3[36]" #print dddstartdatetime[36] #print "dddstartdatetime3[37]" #print dddstartdatetime[37] #if dddstartdatetime[36] == "0" and dddstartdatetime[37] == "0": # print "dddstartdatetime[36] == 0 && dddstartdatetime[37] == 0" # #dendstartdatetime[36] = 1 # #dendstartdatetime[37] = 2 # dddstartdatetimefix = dddstartdatetime[0:36] + "1" + "2" + dddstartdatetime[38:] # # print dddstartdatetimefix # print dddstartdatetime # dddstartdatetime = dddstartdatetimefix #else: # print "notsd00" # pass #remove ":" between time parse timezconv print "descriptionb4escape1" print description #add description time to new description result #desctime = re.escape(desctime).replace("\ "," ") # + "\r\n" #description.append(desctime) #create full dstart time line DTSTART:20140306T141500Z # #####dstartdatetime = "DTSTART;TyID=Europe/Dublin:" + startdate + timez1 + "\r\n" #decl/ #dstartdatetime = dendstartdatetime print "countfrmstrtb4plus" print countfrmstrt print "desclengthb4plus" print desclength #if desclist item count plus 1 is less then decslist length go ahead #elif venue == "Committee": #desclistdups = desclistdups.replace('\\n \\n','\\n\\n').split("\\n\\n") #.split("\\n \\n") descliststart = 0 for desclistdup in desclistdups: print "desclistdupy" print desclistdup desclistdups[descliststart] = desclistdup.replace('\\n(',' (').replace("\\n"," ") descliststart += 1 i = 0; for desclistdup in desclistdups: #desclist[i] = desclist[i].replace("\\n","").replace("\\","") desclistdups[i] = desclistdups[i].replace("\\n"," ").replace("\\","") print "desclistdupclean" print desclistdup i += 1; i = 0; desclistdupbs = desclistdups descduplen = len(desclistdups) print "descduplenx" print descduplen descduplength = int(descduplen) print "descduplengthx" print descduplength countfrmstrtsubloop = 0 countfrmstrtsubloopb = 0 print "desclistdupsb4" print desclistdups if countfrmstrtplusplus <= desclength: print "countfrmstrtplusplusislessdesclength:" print countfrmstrtplusplus print "desclistscheckplusplus" print desclists desclistdupbs = desclists[countfrmstrtplusplus:] print "desclists[countfrmstrtplusplus:]" print desclistdupbs #clean #timecheckfornexttime that is the end time for desclistdupb in desclistdupbs: print "dstartdatetimedd" print dstartdatetime print "startdatedd" print startdate print "dendstartdatetimedd" print dendstartdatetime dstartdatetime = dendstartdatetime # + "\r\n" # timecheckfornexttime = desclistdupb print "desclistdupb" print desclistdupb #if ((not timecheckfornexttime[3].isdigit()) or (timecheckfornexttime [0:4] == "2011") or (timecheckfornexttime [0:4]) == "2012" or (timecheckfornexttime [0:4] != "2013") or (timecheckfornexttime [0:4] == "2014") or (timecheckfornexttime [0:4] == "2015")): print "len(timecheckfornexttime)" print len(timecheckfornexttime) print timecheckfornexttime[0:4] print timecheckfornexttime[0:4] if len(timecheckfornexttime) > 5: print "timecheckfornexttime[3]" print timecheckfornexttime[3] #timeconv2fin = datetime.datetime.strptime(enddesctime,"%I.%M%p").strftime("%H%M%SZ") if ( timecheckfornexttime[3].isdigit() and timecheckfornexttime[0:4] != "2011" and timecheckfornexttime[0:4] != "2012" and timecheckfornexttime[0:4] != "2013" and timecheckfornexttime[1:4] != "2014" and timecheckfornexttime[1:4] != "2015" and timecheckfornexttime[1:4] != "2016"): #print timeconv2fin #if ( timecheckfornexttime[3].isdigit() and timecheckfornexttime[0:4] != "2011" and timecheckfornexttime[0:4] != "2012" and timecheckfornexttime[0:4] != "2013" and timecheckfornexttime[1:4] != "2014" and timecheckfornexttime[1:4] != "5"): #for desclistdup in desclistdups: print "descduplength" print descduplength print "countfrmstrtsubloopb" print countfrmstrtsubloopb #if venue == "Dail" or venue == "Seanad": # # pass #elif venue == "Committee": # countfrmstrtsubloopb = # pass #timecheckfornexttime = desclistdups[countfrmstrtsubloopb].strip() print "timecheckfornexttimecheck" print timecheckfornexttime print "desclen" print desclen print "countfrmstrtb4plusplus" print countfrmstrtplusplus print "timecheckfornexttime[0:4]2" print timecheckfornexttime[0:4] #hhhhhhhhhhhhhhhhhhhhhhhhhhhh moved endtime print "passedifor1" print "timecheckfornexttime[3].isdigit():" #print "desclistdups[countfrmstrtloop]" #print desclistdups[countfrmstrtsubloopb] desctimeplusplus = timecheckfornexttime print "desctimeplusplus" print desctimeplusplus desctimeplusplus = timecheckfornexttime # time print "desctimeplusplus" # time print desctimeplusplus # time enddesctime = timecheckfornexttime # time print "enddesctime" # time print enddesctime # time #remove any extra "\n"'s #enddesctimecheck = enddesctime.split(" ") #enddesctimecheck = enddesctimecheck[0].split("\\n") enddesctimecheck = enddesctime.split("\\n") print "enddesctimecheck" # time print enddesctimecheck # time enddesctime = enddesctimecheck[0] enddesctime = str(enddesctime) print "enddesctimefin" # time print enddesctime # time print "enddesctime[0:2]" # time print enddesctime[0:2] # time print "enddesctimefin" # time print enddesctime # time print "enddesctime[1]" # time print enddesctime[1] # time if ( enddesctime[1].isdigit()): if int(enddesctime[0:2]) > 12: enddesctime24m12 = int(enddesctime[0:2]) - 12 print "enddesctime24m12" # time print enddesctime24m12 # time enddesctime = str(enddesctime24m12) + enddesctime[2:] print "enddesctime" # time print enddesctime # time else: pass else: pass print "enddesctimefixed" # time print enddesctime # time endtimeconv2fin = timefix(enddesctime) timeconv2fin = str(endtimeconv2fin) print "timeconvam2[11:19]" # time print timeconv2fin # time timez = str(timeconv2fin) print "timez2" print timez timezcheck = timez.replace(":",".") #[11:19] print "timezcheck2" print timezcheck #if timezcheck = "12noon" if timezcheck and not timezcheck.isspace(): print "not timezcheck2" pass else: print "defaulttime2" timezcheck = "120000" #timezcheck = "120000" timez = dstartdate + timezcheck # + "Z" #fixx print "timez2endtimef" print timez #timez2endtimef #20151209T004500Z print "timez[9]" print timez[9] print "timez[10]" print timez[10] if timez[9] == "0" and timez[10] == "0": print "timez[9] == 0 && timez2endtimef[10] == 0" timezfix = timez[0:9] + "1" + "2" + timez[11:] #timez[10] = timez[0:9] + "2" + time[11:] print timezfix print timez timez = timezfix else: print "not00" pass #ddstartdatetime = "DTSTART;TdID=Europe/Dublin:" + timez #print "ddstartdatetime4" #print ddstartdatetime #enddatetime = "DTEND;TZID=Europe/Dublin:" + timez + "\r\n" #dupb/ denddatetime = "DTEND;TZID=Europe/Dublin:" + timez + "\r\n" #denddatetime = "DTEND:" + timez + "\r\n" print "denddatetime2f" print denddatetime (category, location, catloc) = locurlcatfunction(locationline) category = category.strip() #startdatetimefull = "DTSTART;TZID=Europe/Dublin:" + startdate + #startdatetimefull = dendstartdatetime print "denddateb4" print denddate startdatetimetc = ddddstartdatetime print "startdatetimetcb4" print startdatetimetc #startdatetimetc = dstartdatetime #full #startdatetimetc = dendstartdatetime print "startdatetimetca4" print startdatetimetc stimecodeconv = timecodefunction(startdatetimetc) print "stimecodeconv" print stimecodeconv description = catloc + " " + stimecodeconv + " " + description #"dupb" + dendstartdatetime = "DTSTART;TZID=Europe/Dublin:" + timez + "\r\n" #dendstartdatetime = "DTSTART:" + timez + "\r\n" print "denddateh" print denddate #dendstartdatetime = denddate + timez # "DTSTART;TZID=Europe/Dublin:" + startdatetime + "\r\n" #d print "dendstartdatetimedt" print dendstartdatetime #print "dendstartdatetime[36]" #print dendstartdatetime[36] #print "dendstartdatetime[37]" #print dendstartdatetime[37] #if dendstartdatetime[36] == "0" and dendstartdatetime[37] == "0": # print "dendstartdatetime[36] == 0 && dendstartdatetime[37] == 0" # #dendstartdatetime[36] = 1 # #dendstartdatetime[37] = 2 # dendstartdatetimefix = dendstartdatetime[0:35] + "1" + "2" + dendstartdatetime[38:] # # print dendstartdatetimefix # print dendstartdatetime # dendstartdatetime = dendstartdatetimefix #else: # print "notd00" # pass #break # why/ else: #pass #desclistdupbnotanumber print "noendtime" #nexendtime print "dorigenddatetime" print dorigenddatetime denddatetime = dorigenddatetime #+ "og" # original endtime pass else: #timechecknexttime is not greater then 2 so what can I do? move to next till pass out of loop pass countfrmstrtsubloopb = countfrmstrtsubloopb + 1 #countfrmstrtsubloop = countfrmstrtsubloop + 1 #else: #notnumber (category, location, catloc) = locurlcatfunction(locationline) category = category.strip() #dstartdatetime = "DTSTART;TZID=Europe/Dublin:" + startdatetime #ddstartdatetime = denddatetime startdatetimetc = dddstartdatetime #function is changing it print "dddstartdatetime5" print dddstartdatetime print "startdatetimetcb5" print startdatetimetc #startdatetimetc = dstartdatetime #full #startdatetimetc = ddendstartdatetime stimecodeconv = timecodefunction(startdatetimetc) print "stimecodeconv" print stimecodeconv description = catloc + " " + stimecodeconv + " " + description #"dupb" + print "countfrmstrtpluscheck" print countfrmstrtplus print "desclists[countfrmstrtplus" print desclists[countfrmstrtplus] #category = category.strip() description = catloc + " " + stimecodeconv + " " + desclists[countfrmstrtplus] # desclistdupb #"last" + descriptionsean = catloc + " " + stimecodeconv + " " + desclists[countfrmstrtplus] #desclistdupb descriptionelsedail = catloc + " " + stimecodeconv + " " + desclists[countfrmstrtplus] #desclistdupb descriptioncomm = catloc + " " + stimecodeconv + " " + desclists[countfrmstrtplus] #desclistdupb ##summaryline = eventlines[10] ##summary = summaryfunction(summaryline) #summary = description print "descriptioncatloc + " " + stimecodeconv + " " + desclists[countfrmstrtplus]" print description dstartdatetime = dddstartdatetime #category = catloc + "\r\n" #pass #break urlnote = "fulloop " newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnote) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnewline) print "first2urlnote" #writeusingfunction oircalf writename = "/var/www/vhosts/thecatchment.org/html/oircal/oircalf.ics" writefullname = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfull.ics" writefullunction(writefullname) writefunction(writename) #open(writename, 'a').writelines("nfirst2urlnote") #el print "venueif" print venue if venue == "Seanad": print "venueifseanad" print venue urlnote = "neasloop " print urlnote newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnote) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnewline) #writeusingfunction oircalf writename = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfsean.ics" writefullname = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfullsean.ics" description = descriptionsean writefunction(writename) writefullunction(writefullname) #open(writename, 'a').writelines("nfirst2urlnotesean") elif venue == "Committee": urlnote = "mmocloop " print urlnote newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnote) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnewline) #writeusingfunction oircalf writename = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfcomm.ics" writefullname = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfullcomm.ics" description = descriptioncomm writefunction(writename) writefullunction(writefullname) #open(writename, 'a').writelines("ncommfirst2urlnote") else: # venue == "Dail": print "venueifdail" print venue urlnote = "laidloop " print urlnote newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnote) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnewline) #writeusingfunction oircalf writename = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfdail.ics" writefullname = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfulldail.ics" description = descriptionelsedail writefunction(writename) writefullunction(writefullname) #open(writename, 'a').writelines("nfirst2urlnoteelsedail") countfrmstrtsubloop = countfrmstrtsubloop + 1 countfrmstrt += 1 else: print "nocountfrmstrtplusplus" # #all in one file print "descduplenxb" print descduplen pass #elsenotanumber else: print "elsenotanumber" print "countfrmstrt5" print countfrmstrt descevent = desclists[countfrmstrt].strip() #descevent = desclist #descevent = desclists print "desceventpluse" print descevent print "dstartdatetimefull2nd" print dstartdatetime startdatetimetc = dstartdatetime print "startdatetimetc2" print startdatetime #dendstartdatetime = "DTSTART;TZID=Europe/Dublin:" + denddate + timezcheck stimecodeconv = timecodefunction(startdatetimetc) #description = catloc + "elsecc" + stimecodeconv + " " + description (category, location, catloc) = locurlcatfunction(locationline) category = category.strip() #category = catloc + "\r\n" description = catloc + " " + stimecodeconv + " " + descevent; # "declelse" descriptioncomm = catloc + " " + stimecodeconv + " " + descevent; descriptionsean = catloc + " " + stimecodeconv + " " + descevent; descriptiondail = catloc + " " + stimecodeconv + " " + descevent; print "catloc + " " + stimecodeconv + " " + descevent" print description ##summaryline = eventlines[10] ##summary = summaryfunction(summaryline) #summary = description print "summaryxdescdesc" print summary summarykeep = summary #dstarttime = startdatetimefull #descevent = desclists[countfrmstrt].strip() #description = descevent #summary = description #category = category.strip() #category = catloc + "\r\n" urlnote = "fulloop " newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnote) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnewline) print "thirdurlnote" #writeusingfunction oircalf writename = "/var/www/vhosts/thecatchment.org/html/oircal/oircalf.ics" writefullname = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfull.ics" writefunction(writename) writefullunction(writefullname) #open(writename, 'a').writelines("nthirdurlnote") print "venue" print venue if venue == "Seanad": print "venueifseanad" print venue urlnote = "neasloop " print urlnote newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnote) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnewline) #writeusingfunction oircalf writename = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfsean.ics" writefullname = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfullsean.ics" description = descriptionsean writefunction(writename) writefullunction(writefullname) #open(writename, 'a').writelines("nseanthirdurlnote") elif venue == "Committee": urlnote = "mmocloop " print urlnote newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnote) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnewline) #writeusingfunction oircalf writename = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfcomm.ics" writefullname = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfullcomm.ics" description = descriptioncomm writefunction(writename) writefullunction(writefullname) #open(writename, 'a').writelines("ndailthirdurlnote") else: # venue == "Dail": print "venueifdail" print venue urlnote = "laidloop " print urlnote newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnote) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnewline) #writeusingfunction oircalf writename = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfdail.ics" writefullname = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfulldail.ics" description = descriptiondail writefunction(writename) writefullunction(writefullname) #open(writename, 'a').writelines("nelsedailthirdurlnote") else: #timechecknexttime is not greater then 2 so what can I do? move to next till pass out of loop pass print "desclistdupnotadigit" print desclistdups #description = "else" #description = desclist.replace("\\n","").replace("\\","") print "notadigit" #countfrmstrt += 1 #description = "beforeurlnotefulloop" #description = desclist.replace("\\n","").replace("\\","") print "desclistdups" print desclistdups if len(desclistdups) > 0: #if desclistdups[0]: del desclistdups[0] print "desclistdupsDELFIRST" print desclistdups else: print "desclistdupspass" print desclistdups pass print "desclistdups" print desclistdups #go to next item in description list #countfrmstrt += 1 print "countfrmstrt" print countfrmstrt pass else: print "nocountfrmstrtplus" #pass #all in one file countfrmstrt += 1 else: description = eventlines[9] print "countfrmstrtisnotlessdesclength:" print "descriptioncountfrmstrtisnotlessdesclength:" print description #pass #end if only one item in description go straight to outputting event to file as is #else: urlnote = "notgreater then one" print urlnote ##summaryline = eventlines[10] # + "\r\n" ##summary = summaryfunction(summaryline) descriptionline = "test" # eventlines[9] descriptionline = filter(None, descriptionline) ###description = eventlines[9] writefullname = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfullsean.ics" writefullunction(writefullname) writename = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfsean.ics" writefunction(writename) ###descsplit = description.split(':') ###description = descsplit[1] ##print "descsplitorig" ##print descsplit descevent = description #description = descriptionfunction(descriptionline) print "descriptionurlnotenotgreater then one" print description locationline = eventlines[8] #+ "\r\n"\n #(urlline, urlnote = "onlyloop " newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnote) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnewline) #newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalf.ics', 'a').writelines(start) #LOCATION;LANGUAGE=en-ie:Dáil Chamber, Leinster House print "startdatetimefull" print startdatetimefull startdatetimetc = startdatetimefull print "startdatetimetc" print startdatetime (category, location, catloc) = locurlcatfunction(locationline) category = category.strip() stimecodeconv = timecodefunction(startdatetimetc) description = catloc + " " + stimecodeconv + " " + descevent #"last" + descriptiondail = catloc + " " + stimecodeconv + " " + descevent #"elsecc" + descriptioncomm = catloc + " " + stimecodeconv + " " + descevent descriptionsean = catloc + " " + stimecodeconv + " " + descevent #category = catloc + "\r\n" print "timeplusdescription" print description startdatetime = startdatetimefull print "startdatetimec" print startdatetime #writeusingfunction oircalf writename = "/var/www/vhosts/thecatchment.org/html/oircal/oircalf.ics" writefunction(writename) writefullname = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfull.ics" writefullunction(writefullname) #open(writename, 'a').writelines("nelsenotgreater then one") #pass if venue == "Seanad": ##summaryline = eventlines[10] # + "\r\n" ##summary = summaryfunction(summaryline) description = descriptionsean #eventlines[9] #description = descriptionfunction(descriptionline) urlnote = "seanloop " print urlnote newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnote) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnewline) #LOCATION;LANGUAGE=en-ie:Dáil Chamber, Leinster House #(urlline, #writeusingfunction oircalf writefullname = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfullsean.ics" writefullunction(writefullname) writename = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfsean.ics" writefunction(writename) #open(writename, 'a').writelines("nseannotgreater then one") #pass elif venue == "Committee": ##summaryline = eventlines[10] # + "\r\n" ##summary = summaryfunction(summaryline) description = descriptioncomm #eventlines[9] #description = descriptionfunction(descriptionline) urlnote = "commloop " print urlnote newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnote) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnewline) #LOCATION;LANGUAGE=en-ie:Dáil Chamber, Leinster House #(urlline, #writeusingfunction oircalf writename = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfcomm.ics" writefunction(writename) writefullname = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfullcomm.ics" writefullunction(writefullname) #open(writename, 'a').writelines("ncommnotgreater then one") #pass else: #if venue == "Dail": #elseloop ##summaryline = eventlines[10] # + "\r\n" ##summary = summaryfunction(summaryline) summary = descriptiondail #eventlines[9] #description = descriptionfunction(descriptionline) print "descriptiondaillelseloop" print description #+ "\r\n"\n #(urlline, urlnote = "dailloop " print urlnote newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnote) newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnewline) #writeusingfunction oircalf writefullname = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfulldail.ics" writename = "/var/www/vhosts/thecatchment.org/html/oircal/oircalfdail.ics" writefullunction(writefullname) writefunction(writename) #open(writename, 'a').writelines("nelsedailnotgreater then one") #pass #else: #urlnote = "elsefail " #newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnote) #newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfurlswork.txt', 'a').writelines(urlnewline) #pass else: print "not within two weeks" print "ddstartdateslash" print ddstartdateslash print "startdatem2w" print startdatem2w print "ddstartdateslash" print ddstartdateslash print "startdatep2w" print startdatep2w pass else: print "duplicate" pass previouseventline9 = eventlines[9] #i+= 1 j += 1 #write ending of entire calendar bottom = "END:VCALENDAR" #write into each file committee/seanad/dail newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalf.ics', 'a').writelines(bottom) #fortwiter newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfcomm.ics', 'a').writelines(bottom) #fortwiter newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfsean.ics', 'a').writelines(bottom) #fortwiter newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfdail.ics', 'a').writelines(bottom) #fortwiter newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfull.ics', 'a').writelines(bottom) #fortwiter newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfullcomm.ics', 'a').writelines(bottom) #fortwiter newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfullsean.ics', 'a').writelines(bottom) #fortwiter newfile = open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfulldail.ics', 'a').writelines(bottom) #fortwiter #this was to perserve previous cron'd file if there was problems but i may have undone it by writing files above newfilename = "/var/www/vhosts/thecatchment.org/html/oircal/newfilenamef.ics" #'newfilename.ics' newfilename = "/var/www/vhosts/thecatchment.org/html/oircal/newfilenamefcomm.ics" #'newfilename.ics' newfilename = "/var/www/vhosts/thecatchment.org/html/oircal/newfilenamefsean.ics" #'newfilename.ics' newfilename = "/var/www/vhosts/thecatchment.org/html/oircal/newfilenamefdail.ics" #'newfilename.ics' #write all into one file with open('/var/www/vhosts/thecatchment.org/html/oircal/oircalf.ics', 'rb') as f: data = f.read() with open('/var/www/vhosts/thecatchment.org/html/oircal/newfilenamef.ics', 'wb') as f: f.write(data.replace('\n\n', '\r\n')) os.unlink('/var/www/vhosts/thecatchment.org/html/oircal/oircalf.ics') os.rename('/var/www/vhosts/thecatchment.org/html/oircal/newfilenamef.ics', '/var/www/vhosts/thecatchment.org/html/oircal/oircalf.ics') #committee with open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfcomm.ics', 'rb') as f: data = f.read() with open('/var/www/vhosts/thecatchment.org/html/oircal/newfilenamefcomm.ics', 'wb') as f: f.write(data.replace('\n\n', '\r\n')) os.unlink('/var/www/vhosts/thecatchment.org/html/oircal/oircalfcomm.ics') os.rename('/var/www/vhosts/thecatchment.org/html/oircal/newfilenamefcomm.ics', '/var/www/vhosts/thecatchment.org/html/oircal/oircalfcomm.ics') #seanad with open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfsean.ics', 'rb') as f: data = f.read() with open('/var/www/vhosts/thecatchment.org/html/oircal/newfilenamefsean.ics', 'wb') as f: f.write(data.replace('\n\n', '\r\n')) os.unlink('/var/www/vhosts/thecatchment.org/html/oircal/oircalfsean.ics') os.rename('/var/www/vhosts/thecatchment.org/html/oircal/newfilenamefsean.ics', '/var/www/vhosts/thecatchment.org/html/oircal/oircalfsean.ics') #dail with open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfdail.ics', 'rb') as f: data = f.read() with open('/var/www/vhosts/thecatchment.org/html/oircal/newfilenamefdail.ics', 'wb') as f: f.write(data.replace('\n\n', '\r\n')) os.unlink('/var/www/vhosts/thecatchment.org/html/oircal/oircalfdail.ics') os.rename('/var/www/vhosts/thecatchment.org/html/oircal/newfilenamefdail.ics', '/var/www/vhosts/thecatchment.org/html/oircal/oircalfdail.ics') with open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfull.ics', 'rb') as full: data = full.read() with open('/var/www/vhosts/thecatchment.org/html/oircal/newfullilenamefull.ics', 'wb') as full: full.write(data.replace('\n\n', '\r\n')) os.unlink('/var/www/vhosts/thecatchment.org/html/oircal/oircalfull.ics') os.rename('/var/www/vhosts/thecatchment.org/html/oircal/newfullilenamefull.ics', '/var/www/vhosts/thecatchment.org/html/oircal/oircalfull.ics') #committee with open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfullcomm.ics', 'rb') as full: data = full.read() with open('/var/www/vhosts/thecatchment.org/html/oircal/newfullilenamefullcomm.ics', 'wb') as full: full.write(data.replace('\n\n', '\r\n')) os.unlink('/var/www/vhosts/thecatchment.org/html/oircal/oircalfullcomm.ics') os.rename('/var/www/vhosts/thecatchment.org/html/oircal/newfullilenamefullcomm.ics', '/var/www/vhosts/thecatchment.org/html/oircal/oircalfullcomm.ics') #seanad with open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfullsean.ics', 'rb') as full: data = full.read() with open('/var/www/vhosts/thecatchment.org/html/oircal/newfullilenamefullsean.ics', 'wb') as full: full.write(data.replace('\n\n', '\r\n')) os.unlink('/var/www/vhosts/thecatchment.org/html/oircal/oircalfullsean.ics') os.rename('/var/www/vhosts/thecatchment.org/html/oircal/newfullilenamefullsean.ics', '/var/www/vhosts/thecatchment.org/html/oircal/oircalfullsean.ics') #dail with open('/var/www/vhosts/thecatchment.org/html/oircal/oircalfulldail.ics', 'rb') as full: data = full.read() with open('/var/www/vhosts/thecatchment.org/html/oircal/newfullilenamefulldail.ics', 'wb') as full: full.write(data.replace('\n\n', '\r\n')) os.unlink('/var/www/vhosts/thecatchment.org/html/oircal/oircalfulldail.ics') os.rename('/var/www/vhosts/thecatchment.org/html/oircal/newfullilenamefulldail.ics', '/var/www/vhosts/thecatchment.org/html/oircal/oircalfulldail.ics')