A macro program is nothing more than a program which automates repetitive tasks. The best thing you could do is learn regular expressions, a little bit of python, and really get to know urllib.
Here's something I whipped up in about 40 minutes:
The output looks like this:
Because the webpage limits the amount of connections I can do in a period of time, I can nab one of these every 5 seconds. Come back later and I'll have them all.
Here's something I whipped up in about 40 minutes:
Code:
import urllib2
import re
import time
BASEURL = 'http://www.crematorydirectory.com/profiles/'
def GetFirmName(webPage):
firmName = re.findall(r'firm_name">(.*?)</span>', webPage, re.DOTALL)
if len(firmName) == 0:
print(webPage)
raise Exception("No firm name?")
else:
return firmName
def GetContactDetails(webPage):
contactDetails = re.findall(r'contact_info">(.*?)</span>', webPage)
if len(contactDetails) == 0:
print(contactDetails)
raise Exception("No contact details?")
return contactDetails
def GetAddresses(baseWebPage, baseURL, curAddresses, state, recursionDepth):
locations = re.findall(r'state_profile.*?href="([a-z/_\.]*?)">([A-Za-z0-9\-]*?)</a>', baseWebPage)
if locations == []:
#We've now reached the base - no sublocations. Get the address information
name = GetFirmName(baseWebPage)
contactDetails = GetContactDetails(baseWebPage)
return '\n'.join(name + contactDetails)
else:
#Not at the base yet. Continue recursion
for url, location in locations:
print(url, location)
time.sleep(5)
locationIndex = urllib2.urlopen(baseURL + url)
locationIndexHTML = locationIndex.read()
locationIndex.close()
#Change URL to not have additional file name on the end
urlwithoutfilename = '/'.join(url.split('/')[:-1])
if len(urlwithoutfilename) > 0 and urlwithoutfilename[-1] != '/':
urlwithoutfilename += '/'
curAddresses.append(GetAddresses(locationIndexHTML, baseURL + urlwithoutfilename, curAddresses, location, recursionDepth + 1))
return curAddresses
baseIndex = urllib2.urlopen(BASEURL + 'index.html')
baseIndexHTML = baseIndex.read()
baseIndex.close()
allAddresses = GetAddresses(baseIndexHTML, BASEURL, [], '', 0)
for eachThing in allAddresses:
if type(eachThing) != type(''):
allAddresses.remove(eachThing)
print(allAddresses)
print('\n\n\n'.join(allAddresses))
f = open('outputoutput.txt', 'wb')
f.write('\n\n\n'.join(allAddresses))
f.close()The output looks like this:
Spoiler
Because the webpage limits the amount of connections I can do in a period of time, I can nab one of these every 5 seconds. Come back later and I'll have them all.