import re, os, urllib2, time, datetime, collections data_oldss = [line.split('|') for line in '''\ -1|WORLD|69|67|71.1|7323187457|- -20|EUROPEAN UNION|80.2|77.4|83.2|515052778|- 20|China|75.5|73.5|77.9|1373541278|ea 10|India|68.5|67.3|69.8|1266883598|as 25|USA|79.8|77.5|82.1|323995528|na |Indonesia|72.7|70.1|75.5|258316051|ea |Brazil|73.8|70.2|77.5|205823665|sa |Pakistan|67.7|65.8|69.8|201995540|as -20|Nigeria|53.4|52.4|54.5|186053386|af |Bangladesh|73.2|71|75.4|156186882|as -10|Russia|70.8|65|76.8|142355415|eu 1|Japan|85|81.7|88.5|126702133|ea |Mexico|75.9|73.1|78.8|123166749|na |Philippines|69.2|65.7|72.9|102624209|ea |Ethiopia|62.2|59.8|64.7|102374044|af |Vietnam|73.4|70.9|76.2|95261021|ea |Egypt|72.7|71.4|74.2|94666993|af |Iran|71.4|69.8|73.1|82801633|me -15|DR Congo|57.3|55.8|58.9|81331050|af |Germany|80.7|78.4|83.1|80722792|eu |Turkey|74.8|72.5|77.3|80274604|me |Thailand|74.7|71.5|78|68200824|ea |France|81.8|78.7|85.1|66836154|eu 12|UK|80.7|78.5|83|64430428|eu |Italy|82.2|79.6|85|62007540|eu |Burma|66.6|64.2|69.2|56890418|ea |South Africa|63.1|61.6|64.6|54300704|af |Tanzania|62.2|60.8|63.6|52482726|af |Korea, South|82.4|79.3|85.8|50924172|ea |Spain|81.7|78.7|84.9|48563476|eu |Colombia|75.7|72.6|79|47220856|sa |Kenya|64|62.6|65.5|46790758|af |Ukraine|71.8|67.1|76.9|44209733|eu |Argentina|77.1|74|80.4|43886748|sa |Algeria|76.8|75.5|78.2|40263711|af |Poland|77.6|73.7|81.7|38523261|eu |Uganda|55.4|54|56.9|38319241|af |Iraq|74.9|72.6|77.2|38146025|me |Sudan|64.1|62|66.3|36729501|af |Canada|81.9|79.2|84.6|35362905|na |Morocco|76.9|73.8|80.1|33655786|af -15|Afghanistan|51.3|49.9|52.7|33332025|as |Malaysia|75|72.2|78|30949962|ea |Venezuela|75.8|72.7|78.9|30912302|sa |Peru|73.7|71.7|75.9|30741062|sa |Uzbekistan|73.8|70.7|77|29473614|ca |Nepal|70.7|70.1|71.3|29033914|as |Saudi Arabia|75.3|73.2|77.4|28160273|me |Yemen|65.5|63.4|67.8|27392779|me |Ghana|66.6|64.1|69.1|26908262|af |Mozambique|53.3|52.6|54.1|25930150|af |Korea, North|70.4|66.6|74.5|25115311|ea |Madagascar|65.9|64.4|67.4|24430325|af |Cameroon|58.5|57.1|59.9|24360803|af |Cote d'Ivoire|58.7|57.5|59.9|23740424|af |Taiwan|80.1|77|83.5|23464787|ea |Australia|82.2|79.8|84.8|22992654|oc |Sri Lanka|76.8|73.3|80.4|22235000|as |Romania|75.1|71.7|78.8|21599736|eu |Angola|56|54.8|57.2|20172332|af |Burkina Faso|55.5|53.4|57.6|19512533|af |Niger|55.5|54.3|56.8|18638600|af |Malawi|61.2|59.2|63.2|18570321|af |Kazakhstan|70.8|65.6|75.7|18360353|ca |Chile|78.8|75.7|81.9|17650114|sa |Mali|55.8|53.9|57.7|17467108|af |Syria|74.9|72.5|77.4|17185170|me |Netherlands|81.3|79.2|83.6|17016967|eu |Ecuador|76.8|73.8|79.9|16080778|sa |Cambodia|64.5|62|67.1|15957223|ea |Zambia|52.5|50.8|54.1|15510711|af |Guatemala|72.3|70.3|74.4|15189958|la |Zimbabwe|58|57.3|58.7|14546961|af |Senegal|61.7|59.7|63.8|14320055|af |Rwanda|60.1|58.5|61.7|12988423|af |Guinea|60.6|59|62.2|12093349|af -1|Chad|50.2|49|51.5|11852462|af |Belgium|81|78.4|83.7|11409077|eu |Cuba|78.7|76.4|81.1|11179995|la |Tunisia|76.1|74|78.4|11134588|af |Burundi|60.5|58.8|62.3|11099298|af |Bolivia|69.2|66.4|72.1|10969649|sa |Portugal|79.3|76.1|82.8|10833816|eu |Somalia|52.4|50.3|54.5|10817354|af |Greece|80.5|77.9|83.3|10773253|eu |Benin|61.9|60.5|63.3|10741458|af |Czechia|78.6|75.7|81.8|10644842|eu |Dominican Republic|78.1|75.9|80.5|10606865|la |Haiti|63.8|61.2|66.4|10485800|la |Sweden|82.1|80.2|84.1|9880604|eu |Hungary|75.9|72.2|79.8|9874784|eu |Azerbaijan|72.5|69.5|75.8|9872765|me -17|Belarus|72.7|67.2|78.6|9570376|eu |Honduras|71.1|69.5|72.8|8893259|la |Austria|81.5|78.9|84.3|8711770|eu |Tajikistan|67.7|64.6|71|8330946|ca |Jordan|74.6|73.2|76.1|8185384|me |Switzerland|82.6|80.3|85|8179294|eu |Israel|82.4|80.6|84.4|8174527|me |Togo|65|62.3|67.7|7756937|af |Hong Kong|82.9|80.3|85.8|7167403|ea |Bulgaria|74.5|71.2|78|7144653|eu |Serbia|75.5|72.6|78.5|7143921|eu |Laos|64.3|62.2|66.4|7019073|ea |Paraguay|77.2|74.5|80|6862812|sa |Papua New Guinea|67.2|65|69.5|6791317|ea |Libya|76.5|74.7|78.3|6541948|af |Lebanon|77.6|76.3|78.9|6237738|me |El Salvador|74.7|71.4|78.1|6156670|la |Sierra Leone|58.2|55.6|60.9|6018888|af |Nicaragua|73.2|71.1|75.5|5966798|la |United Arab Emirates|77.5|74.8|80.2|5927482|me |Eritrea|64.9|62.4|67.5|5869869|af 10|Singapore|85|82.3|87.8|5781728|ea |Kyrgyzstan|70.7|66.5|75.1|5727553|ca |Denmark|79.4|77|82|5593785|eu |Central African Republic|52.3|51|53.7|5507257|af |Finland|80.9|77.9|84|5498211|eu |Slovakia|77.1|73.5|80.9|5445802|eu |Turkmenistan|70.1|67.1|73.3|5291317|ca |Norway|81.8|79.8|83.9|5265158|eu |Ireland|80.8|78.5|83.2|4952473|eu |Georgia|76.2|72.1|80.6|4928052|me |Costa Rica|78.6|75.9|81.4|4872543|la |Congo, Republic of the|59.3|58.1|60.6|4852412|af |New Zealand|81.2|79.1|83.3|4474549|oc |Croatia|75.9|72.7|79.2|4313707|eu |Liberia|59|57.3|60.8|4299944|af |Bosnia and Herzegovina|76.7|73.7|80|3861912|eu |Panama|78.6|75.8|81.6|3705246|la |Mauritania|63|60.7|65.4|3677293|af |Puerto Rico|79.4|75.8|83.1|3578056|la |Moldova|70.7|66.9|74.8|3510485|eu |Oman|75.5|73.5|77.5|3355262|me |Uruguay|77.2|74.1|80.5|3351016|sa |Armenia|74.6|71.4|78.3|3051250|me |Albania|78.3|75.7|81.2|3038594|eu |Mongolia|69.6|65.4|74.1|3031330|ea |Jamaica|73.6|72|75.3|2970340|la |Lithuania|74.9|69.5|80.6|2854235|eu |Kuwait|78|76.6|79.4|2832776|me |West Bank|75|73|77.1|2697687|me |Namibia|63.6|62.1|65.1|2436469|af |Qatar|78.7|76.7|80.8|2258283|me 1|Botswana|54.5|56.3|52.6|2209208|af |Macedonia|76.2|73.6|79|2100025|eu |Gambia, The|64.9|62.5|67.3|2009648|af |Slovenia|78.2|74.6|82|1978029|eu |Latvia|74.5|69.9|79.3|1965686|eu |Lesotho|53|52.9|53.1|1953070|af -2|Guinea-Bissau|50.6|48.6|52.7|1759159|af |Gaza Strip|73.9|72.3|75.7|1753327|me |Gabon|52.1|51.6|52.5|1738541|af 1|Swaziland|51.6|52.2|51|1451428|af |Bahrain|78.9|76.7|81.1|1378904|me |Mauritius|75.6|72.2|79.2|1348242|af |Timor-Leste|68.1|66.5|69.7|1261072|ea |Estonia|76.7|71.9|81.7|1258545|eu |Trinidad and Tobago|72.9|69.9|75.9|1220479|la |Cyprus|78.7|75.8|81.6|1205575|eu |Fiji|72.7|70|75.5|915303|oc |Djibouti|63.2|60.7|65.8|846687|af |Comoros|64.2|61.9|66.6|794678|af |Equatorial Guinea|64.2|63.1|65.4|759451|af |Bhutan|70.1|69.1|71.1|750125|as |Guyana|68.4|65.4|71.5|735909|sa |Solomon Islands|75.3|72.7|78.1|635027|oc -10|Macau|84.5|81.6|87.6|597425|ea |Western Sahara|63|60.7|65.4|587020|af |Suriname|72.2|69.8|74.8|585824|sa |Luxembourg|82.3|79.8|84.9|582291|eu |Cabo Verde|72.1|69.8|74.5|553432|af |Brunei|77.2|74.8|79.6|436620|ea |Malta|80.4|78|82.8|415196|eu |Maldives|75.6|73.3|78|392960|as |Belize|68.7|67.2|70.4|353858|la |Iceland|83|80.9|85.3|335878|eu |Bahamas, The|72.4|70|74.8|327316|la |Barbados|75.3|73|77.7|291495|la |French Polynesia|77.2|74.9|79.6|285321|oc |Vanuatu|73.4|71.8|75.1|277554|oc |New Caledonia|77.7|73.7|81.9|275355|oc |Samoa|73.7|70.8|76.8|198926|oc |Sao Tome and Principe|64.9|63.6|66.3|197541|af |Saint Lucia|77.8|75|80.7|164464|la |Guam|79.1|76.1|82.4|162742|oc |Curacao|78.3|76|80.7|149035|la |Aruba|76.8|73.7|79.9|113648|la |Grenada|74.3|71.7|77.1|111219|la |Kiribati|66.2|63.7|68.8|106925|oc |Tonga|76.2|74.7|77.8|106513|oc |Micronesia, Federated States of|72.9|70.8|75|104719|oc |Virgin Islands|80|77|83.2|102951|la |Saint Vincent and the Grenadines|75.3|73.3|77.4|102350|la |Jersey|81.9|79.4|84.5|98069|eu |Antigua and Barbuda|76.5|74.4|78.8|93581|la |Seychelles|74.7|70.2|79.4|93186|af |Isle of Man|81.2|79.5|83|88195|eu |Andorra|82.8|80.6|85.1|85660|eu |Dominica|77|74|80.1|73757|la |Marshall Islands|73.1|70.9|75.4|73376|oc |Bermuda|81.3|78.1|84.5|70537|na |Guernsey|82.5|79.9|85.4|66297|eu |Greenland|72.4|69.7|75.2|57728|na |Cayman Islands|81.2|78.5|84|57268|la |American Samoa|75.4|72.4|78.5|54194|oc |Northern Mariana Islands|78|75.3|80.8|53467|oc |Saint Kitts and Nevis|75.7|73.3|78.2|52329|la |Turks and Caicos Islands|79.8|77.1|82.7|51430|la |Faroe Islands|80.4|77.8|83.1|50456|eu |Sint Maarten|78.1|75.8|80.6|41486|la |Liechtenstein|81.9|79.7|84.6|37937|eu |British Virgin Islands|78.6|77.2|80.1|34232|la |San Marino|83.3|80.7|86.1|33285|eu -1|Monaco|89.5|85.6|93.5|30581|eu |Gibraltar|79.4|76.6|82.5|29328|eu |Palau|73.1|69.9|76.5|21347|oc |Anguilla|81.4|78.8|84.1|16752|la |Wallis and Futuna|79.7|76.7|82.8|15664|oc |Tuvalu|66.5|64.3|68.8|10959|oc |Nauru|67.1|63|70.5|9591|oc |Cook Islands|75.8|73|78.8|9556|oc |Saint Helena, Ascension, and Tristan da Cunha|79.5|76.6|82.6|7795|af |Saint Pierre and Miquelon|80.5|78.2|83|5595|na 1|Montserrat|74.4|75.8|72.9|5267|la |Falkland Islands (Islas Malvinas)|77.9|75.6|79.6|2931|sa |Svalbard|NA|NA|NA|2667|eu |Norfolk Island|NA|NA|NA|2210|oc |Christmas Island|NA|NA|NA|2205|oc |Tokelau|NA|NA|NA|1285|oc |Niue|NA|NA|NA|1190|oc |Cocos (Keeling) Islands|NA|NA|NA|596|oc |Pitcairn Islands|NA|NA|NA|54|oc '''.strip().split('\n')] # do_refresh_cache = True def read_url(url, headers={}, path_cache=None, is_verbose=True): if (path_cache is None): file_cache = os.path.basename(url) path_cache = os.path.join('%s.cache' % (os.path.splitext(__file__)[0]), file_cache if (len(file_cache) > 0) else '%s.htm' % (os.path.basename(url.rstrip('/')))) if (('do_refresh_cache' in globals() and do_refresh_cache) or (not os.path.isfile(path_cache))): request = urllib2.Request(url, headers=headers) try: html = urllib2.urlopen(request).read() except urllib2.HTTPError as e: html = ''; print(e) try: os.makedirs(os.path.dirname(path_cache)) except OSError: pass with open(path_cache, 'wb') as f_html: f_html.write(html) if (is_verbose): print('%s > %s' % (url, path_cache)) time.sleep(1) ## avoid rate-limit-exceeded error else: with open(path_cache) as f_html: html = f_html.read() if (is_verbose): print('< %s' % (path_cache)) try: html = html.decode('utf-8') except UnicodeDecodeError: pass return html def fmt(string): ## string.format(**vars()) using tags {expression!format} by CMG Lee def f(tag): i_sep = tag.rfind('!'); return (re.sub('\.0+$', '', str(eval(tag[1:-1]))) if (i_sep < 0) else ('{:%s}' % tag[i_sep + 1:-1]).format(eval(tag[1:i_sep]))) return (re.sub(r'(?<!{){[^{}]+}', lambda m:f(m.group()), string) .replace('{{', '{').replace('}}', '}')) def append(obj, string): return obj.append(fmt(string)) def format_tab(*arg): return '\t'.join([str(el) for el in (arg if len(arg) > 1 else arg[0])]) def tabbify(cellss, separator='|'): cellpadss = [list(rows) + [''] * (len(max(cellss, key=len)) - len(rows)) for rows in cellss] fmts = ['%%%ds' % (max([len(str(cell)) for cell in cols])) for cols in zip(*cellpadss)] return '\n'.join([separator.join(fmts) % tuple(rows) for rows in cellpadss]) def hex_rgb(colour): ## convert [#]RGB to #RRGGBB and [#]RRGGBB to #RRGGBB return '#%s' % (colour if len(colour) > 4 else ''.join([c * 2 for c in colour])).lstrip('#') def try_int_float(field): try: return int(field) except: try: return float(field) except: return field def roundm(x, multiple=1): try: x[0]; return [roundm(element, multiple) for element in x] ## x[0] checks if x is iterable except: return int(math.floor(float(x) / multiple + 0.5)) * multiple def findall(regex, string): return re.findall(regex, string, flags=re.I|re.DOTALL) def sub(regex_replace, regex_with, string): return str(re.sub(regex_replace, regex_with, string, flags=re.DOTALL).strip()) def make_serial(name): return sub(r'[^a-z]', '', name.lower()) def make_table(datass): return '\n'.join(['|'.join([str(data) for data in datas]) for datas in datass]) data_newss = {} html_expectancy = read_url('http://cia.gov/library/publications/resources/the-world-factbook/fields/355.html') html_expectancyss = findall(r'(<td.+?</td>)\s*(<td.+?</td>)', html_expectancy) for html_expectancys in html_expectancyss: html_divs = findall(r'<div.+?</div>', html_expectancys[1]) name = sub(r'<.*?>', '', html_expectancys[0]) serial = make_serial(name) # expectancy_male = None # expectancy_female = None # try: expectancy_male = float(findall(r'[\d.]+(?= years)', html_divs[1])[0]) # except Exception: pass # try: expectancy_female = float(findall(r'[\d.]+(?= years)', html_divs[2])[0]) # except Exception: pass # if (not serial in data_newss): data_newss[serial] = {} # data_newss[serial]['male' ] = expectancy_male # data_newss[serial]['female'] = expectancy_female try: expectancy_overall = float(findall(r'(?:[\d.]+(?= years)|\d+\.\d+)', html_divs[0])[0]) expectancy_male = float(findall(r'(?:[\d.]+(?= years)|\d+\.\d+)', html_divs[1])[0]) expectancy_female = float(findall(r'(?:[\d.]+(?= years)|\d+\.\d+)', html_divs[2])[0]) if (not serial in data_newss): data_newss[serial] = {} data_newss[serial]['overall'] = expectancy_overall data_newss[serial]['male' ] = expectancy_male data_newss[serial]['female' ] = expectancy_female except Exception: pass html_population = read_url('http://cia.gov/library/publications/resources/the-world-factbook/fields/335.html') html_populationss = findall(r'(<td.+?</td>)\s*(<td.+?</td>)', html_population) for html_populations in html_populationss: name = sub(r'<.*?>', '', html_populations[0]) serial = make_serial(name) # population = None # if (not 'no indigenous' in html_populations[1]): # try: population = int(sub(',','',findall(r'[\d,]+', html_populations[1])[0])) # except Exception: pass # if (not serial in data_newss): data_newss[serial] = {} # data_newss[serial]['population'] = population if (not 'no indigenous' in html_populations[1]): try: population = int(sub(',','',findall(r'[\d,]+', html_populations[1])[0])) if (not serial in data_newss): data_newss[serial] = {} data_newss[serial]['name'] = name data_newss[serial]['population'] = population except Exception: pass outss = [] for serial in sorted(data_newss): data_news = data_newss[serial] try: outss.append([serial, data_news['name'], data_news['population'], data_news['overall'], data_news['male'], data_news['female']]) # data_news['population'] if ('population' in data_news) else None, # data_news['male'] if ('male' in data_news) else None, # data_news['female'] if ('female' in data_news) else None]) except Exception: pass # print(data_newss.pop(serial)) # print(tabbify(outss)) outss = [] # print(tabbify(data_oldss)) map_keeps = {'usa':'unitedstates', 'uk':'unitedkingdom', 'drcongo':'congodemocraticrepublicofthe'} map_changes = {'swaziland':'eswatini'} for data_olds in data_oldss: name = data_olds[1] serial = make_serial(name) data_news = None try: if (serial in map_keeps): serial = map_keeps[serial] if (serial in map_changes): serial = map_changes[serial] data_news = data_newss[serial] name = data_news['name'] else: data_news = data_newss[serial] except Exception: pass outss.append([data_olds[0], name, # data_news['name' ] if ('name' in data_news) else 'NA', data_news['overall' ] if ('overall' in data_news) else 'NA', data_news['male' ] if ('male' in data_news) else 'NA', data_news['female' ] if ('female' in data_news) else 'NA', data_news['population'] if ('population' in data_news) else 'NA', data_olds[6]]) # outss.append(data_olds) if (name != data_news['name']): print(name, data_news['name']) # print(tabbify(outss)) outss = outss[:2] + sorted(outss[2:], key=lambda lines:lines[5], reverse=True) dir_cache = '%s.cache' % (os.path.splitext(__file__)[0]) with open(os.path.join(dir_cache, 'old.txt'), 'w') as f: f.write(make_table(data_oldss)) with open(os.path.join(dir_cache, 'new.txt'), 'w') as f: f.write(make_table(outss))