From 321eba0f7943a7172dbfce6d470145e48ada6397 Mon Sep 17 00:00:00 2001 From: wang--ge Date: Mon, 18 Sep 2023 17:23:17 +0800 Subject: [PATCH] units 2.17 units_cur validate --- units_cur | 322 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 245 insertions(+), 77 deletions(-) diff --git a/units_cur b/units_cur index 00281d8..70c8d8e 100755 --- a/units_cur +++ b/units_cur @@ -28,8 +28,17 @@ from __future__ import absolute_import, division, print_function # # -version = '4.2' +version = '5.0' +# Version 5.0: +# +# Rewrite to support multiple different data sources due to disappearance +# of the Yahoo feed. Includes support for base currency selection. +# +# Version 4.3: 20 July 2018 +# +# Validate rate data from server +# # Version 4.2: 18 April 2018 # # Handle case of empty/malformed entry returned from the server @@ -55,10 +64,17 @@ from sys import exit, stderr, stdout outfile_name = 'currency.units' +# valid metals + +validmetals = ['silver','gold','platinum'] + +PRIMITIVE = '! # Base unit, the primitive unit of currency' + # This exchange rate table lists the currency ISO 4217 codes, their # long text names, and any fixed definitions. If the definition is # empty then units_cur will query the server for a value. +rate_index = 1 currency = OrderedDict([ ('ATS', ['austriaschilling', '1|13.7603 euro']), ('BEF', ['belgiumfranc', '1|40.3399 euro']), @@ -83,13 +99,14 @@ currency = OrderedDict([ ('BGN', ['bulgarialev', '1|1.9558 euro']), ('BAM', ['bosniaconvertiblemark','germanymark']), ('KMF', ['comorosfranc', '1|491.96775 euro']), - ('XOF', ['westafricanfranc', '1|655.957 euro']), + ('XOF', ['westafricafranc', '1|655.957 euro']), ('XPF', ['cfpfranc', '1|119.33 euro']), - ('XAF', ['centralafricancfafranc','1|655.957 euro']), + ('XAF', ['centralafricacfafranc','1|655.957 euro']), ('AED', ['uaedirham','']), ('AFN', ['afghanafghani','']), ('ALL', ['albanialek','']), ('AMD', ['armeniadram','']), + ('ANG', ['antillesguilder','']), ('AOA', ['angolakwanza','']), ('ARS', ['argentinapeso','']), ('AUD', ['australiadollar','']), @@ -109,7 +126,7 @@ currency = OrderedDict([ ('BTN', ['bhutanngultrum','']), ('BWP', ['botswanapula','']), ('BYN', ['belarusruble','']), - ('BYR', ['oldbelarusruble','10000 BYN']), + ('BYR', ['oldbelarusruble','1|10000 BYN']), ('BZD', ['belizedollar','']), ('CAD', ['canadadollar','']), ('CDF', ['drcfranccongolais','']), @@ -127,7 +144,7 @@ currency = OrderedDict([ ('DZD', ['algeriadinar','']), ('EGP', ['egyptpound','']), ('ERN', ['eritreanakfa','']), - ('ETB', ['ethiopianbirr','']), + ('ETB', ['ethiopiabirr','']), ('EUR', ['euro','']), ('FJD', ['fijidollar','']), ('FKP', ['falklandislandspound','']), @@ -164,10 +181,9 @@ currency = OrderedDict([ ('KZT', ['kazakhstantenge','']), ('LAK', ['laokip','']), ('LBP', ['lebanonpound','']), - ('LKR', ['srilankanrupee','']), + ('LKR', ['srilankarupee','']), ('LRD', ['liberiadollar','']), - ('LTL', ['lithuanialita','']), - ('LVL', ['latvialat','']), + ('LSL', ['lesotholoti','']), ('LYD', ['libyadinar','']), ('MAD', ['moroccodirham','']), ('MDL', ['moldovaleu','']), @@ -176,13 +192,14 @@ currency = OrderedDict([ ('MMK', ['myanmarkyat','']), ('MNT', ['mongoliatugrik','']), ('MOP', ['macaupataca','']), - ('MRO', ['mauritaniaouguiya','']), + ('MRO', ['mauritaniaoldouguiya','1|10 MRU']), + ('MRU', ['mauritaniaouguiya', '']), ('MUR', ['mauritiusrupee','']), ('MVR', ['maldiverufiyaa','']), ('MWK', ['malawikwacha','']), ('MXN', ['mexicopeso','']), ('MYR', ['malaysiaringgit','']), - ('MZN', ['mozambicanmetical','']), + ('MZN', ['mozambiquemetical','']), ('NAD', ['namibiadollar','']), ('NGN', ['nigerianaira','']), ('NIO', ['nicaraguacordobaoro','']), @@ -212,7 +229,9 @@ currency = OrderedDict([ ('SLL', ['sierraleoneleone','']), ('SOS', ['somaliaschilling','']), ('SRD', ['surinamedollar','']), - ('STD', ['saotome&principedobra','']), + ('SSP', ['southsudanpound','']), + ('STD', ['saotome&principeolddobra','']), + ('STN', ['saotome&principedobra','']), ('SVC', ['elsalvadorcolon','']), ('SYP', ['syriapound','']), ('SZL', ['swazilandlilangeni','']), @@ -227,15 +246,15 @@ currency = OrderedDict([ ('TZS', ['tanzaniashilling','']), ('UAH', ['ukrainehryvnia','']), ('UGX', ['ugandaschilling','']), - ('USD', ['unitedstatesdollar', 'US$']), + ('USD', ['US$', '']), ('UYU', ['uruguaypeso','']), ('UZS', ['uzbekistansum','']), - ('VEF', ['venezuelabolivar','']), - ('VEB', ['venezuelaoldbolivar', '1000 VEF']), + ('VEF', ['venezuelabolivarfuerte','']), + ('VES', ['venezuelabolivarsoberano','']), ('VND', ['vietnamdong','']), ('VUV', ['vanuatuvatu','']), ('WST', ['samoatala','']), - ('XAF', ['centralafricancfafranc','']), + ('XAF', ['centralafricacfafranc','']), ('XCD', ['eastcaribbeandollar','']), ('XDR', ['specialdrawingrights','']), ('YER', ['yemenrial','']), @@ -244,6 +263,147 @@ currency = OrderedDict([ ('ZWL', ['zimbabwedollar','']), ]) +def validfloat(x): + try: + float(x) + return True + except ValueError: + return False + +def addrate(verbose,form,code,rate): + if code not in currency.keys(): + if (verbose): + stderr.write('Got unknown currency with code {}\n'.format(code)) + else: + if not currency[code][rate_index]: + if validfloat(rate): + currency[code][rate_index] = form.format(rate) + else: + stderr.write('Got invalid rate "{}" for currency "{}"\n'.format( + rate, code)) + elif verbose: + if currency[code][rate_index] != form.format(rate): + stderr.write('Got value "{}" for currency "{}" but ' + 'it is already defined as {}\n'.format(rate, code, + currency[code][rate_index])) + +def getjson(address,args=None): + try: + res = requests.get(address,args) + res.raise_for_status() + return(res.json()) + except requests.exceptions.RequestException as e: + stderr.write('Error connecting to currency server:\n{}.\n'.format(e)) + exit(1) + +######################################################## +# +# Connect to floatrates for currency update +# + +def floatrates(verbose,base,dummy): + webdata = getjson('https://www.floatrates.com/daily/'+base+'.json') + for index in webdata: + entry = webdata[index] + if 'rate' not in entry or 'code' not in entry: # Skip empty/bad entries + if verbose: + stderr.write('Got bad entry from server: '+str(entry)+'\n') + else: + addrate(verbose,'{} '+base,entry['code'],entry['inverseRate']) + currency[base][rate_index] = PRIMITIVE + return('FloatRates ('+base+' base)') + +######################################################## +# +# Connect to European central bank site +# + +def eubankrates(verbose,base,dummy): + if verbose and base!='EUR': + stderr.write('European bank uses euro for base currency. Specified base {} ignored.\n'.format(base)) + import xml.etree.ElementTree as ET + try: + res=requests.get('https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml') + res.raise_for_status() + data = ET.fromstring(res.content)[2][0] + except requests.exceptions.RequestException as e: + stderr.write('Error connecting to currency server:\n{}.\n'. + format(e)) + exit(1) + for entry in data.iter(): + if entry.get('time'): + continue + rate = entry.get('rate') + code = entry.get('currency') + if not rate or not code: # Skip empty/bad entries + if verbose: + stderr.write('Got bad entry from server, code {} and rate {}\n'.format(code,rate)) + else: + addrate(verbose,'1|{} euro', code, rate) + currency['EUR'][rate_index]=PRIMITIVE + return('the European Central Bank (euro base)') + +######################################################## +# +# Connect to fixer.io (requires API key) +# +# Free API key does not allow changing base currency +# With free key only euro base is supported, and https is not allowed +# + +def fixer(verbose,base,key): + if not key: + stderr.write('API key required for this source\n') + exit(1) + if verbose and base!='EUR': + stderr.write('Fixer uses euro for base currency. Specified base {} ignored.\n'.format(base)) + webdata = getjson('http://data.fixer.io/api/latest', {'access_key':key}) + if not webdata['success']: + stderr.write('Currency server error: '+webdata['error']['info']) + exit(1) + for code in webdata['rates']: + addrate(verbose,'1|{} euro', code, webdata['rates'][code]) + currency['EUR'][rate_index] = PRIMITIVE + return('Fixer (euro base)') + +######################################################## +# +# Connect to openexchangerates (requires API key) +# +# Free API key does not allow changing the base currency +# + +def openexchangerates(verbose,base,key): + if not key: + stderr.write('API key required for this source\n') + exit(1) + if verbose and base!='USD': + stderr.write('Open Exchange Rates uses US dollar for base currency. Specified base {} ignored.\n'.format(base)) + webdata = getjson('https://openexchangerates.org/api/latest.json', + {'app_id':key} + ) + for code in webdata['rates']: + addrate(verbose,'1|{} US$', code, webdata['rates'][code]) + currency['USD'][rate_index] = PRIMITIVE + return('open exchange rates (USD base)') + +####################################################### +# +# list of valid source names and corresponding functions +# + +sources = { + 'floatrates': floatrates, + 'eubank' : eubankrates, + 'fixer' : fixer, + 'openexchangerates': openexchangerates, +} + +####################################################### +# +# Argument Processing +# + ap = ArgumentParser( description="Update currency information for 'units' " "into the specified filename or if no filename is " @@ -271,64 +431,43 @@ ap.add_argument('-v','--verbose', help='display details when fetching currency data', ) -outfile_name = ap.parse_args().output_file -verbose = ap.parse_args().verbose +ap.add_argument('-s','--source',choices=list(sources.keys()), + default='floatrates', + help='set currency data source', +) -try: - res = requests.get('http://finance.yahoo.com/webservice/v1/symbols' - '/allcurrencies/quote?format=json') - res.raise_for_status() - webdata = res.json()['list']['resources'] -except requests.exceptions.RequestException as e: - stderr.write('Error connecting to currency server:\n{}.\n'. - format(e)) +ap.add_argument('-b','--base',default='USD', + help='set the base currency (when allowed by source). BASE should be a 3 letter ISO currency code, e.g. USD. The specified currency will be the primitive currency unit used by units. Only the floatrates source supports this option.', +) + +ap.add_argument('-k','--key',default='', + help='set API key for sources that require it' +) + +args = ap.parse_args() +outfile_name = args.output_file +verbose = args.verbose +source = args.source +base = args.base +apikey = args.key + +if base not in currency.keys(): + stderr.write('Base currency {} is not a known currency code.\n'.format(base)) exit(1) - -rate_index = 1 -for data in webdata: - entry = data['resource']['fields'] - if 'price' not in entry or 'symbol' not in entry: # Skip empty/bad entries - if verbose: - stderr.write('Got bad entry from server: '+str(entry)+'\n') - else: - rate = entry['price'] - code = entry['symbol'][0:3] - if code not in currency.keys(): - if (verbose): - stderr.write('Got unknown currency with code {}\n'.format(code)) - else: - if not currency[code][rate_index]: - currency[code][rate_index] = '1|{} US$'.format(rate) - elif verbose: - stderr.write('Got value "{}" for currency "{}" but ' - 'it is already defined\n'.format(rate, code)) +######################################################## +# +# Fetch currency data from specified curerncy source +# - +sourcename = sources[source](verbose,base,apikey) + # Delete currencies where we have no rate data -for code in currency.keys(): +for code in list(currency.keys()): if not currency[code][rate_index]: if verbose: - stderr.write('No data for {}'.format(code)) + stderr.write('No data for {}\n'.format(code)) del currency[code] - -try: - req = requests.get('http://services.packetizer.com/spotprices/?f=json') - req.raise_for_status() - metals = req.json() -except requests.exceptions.RequestException as e: - stderr.write('Error connecting to spotprices server:\n{}\n'.format(e)) - exit(1) - -del metals['date'] - -try: - req = requests.get('http://services.packetizer.com/btc/?f=json') - req.raise_for_status() - bitcoin = req.json() -except requests.exceptions.RequestException as e: - stderr.write('Error connecting to bitcoin server:\n{}\n'.format(e)) - exit(1) cnames = [currency[code][0] for code in currency.keys()] crates = [currency[code][1] for code in currency.keys()] @@ -336,40 +475,69 @@ crates = [currency[code][1] for code in currency.keys()] codestr = '\n'.join('{:23}{}'. format(code, name) for (code,name) in zip(currency.keys(), cnames)) -datestr = date.today().isoformat() - maxlen = max(len(name) for name in cnames) + 2 ratestr = '\n'.join( '{:{}}{}'.format(name, maxlen, rate) for (name, rate) in zip(cnames, crates) ) -ozzystr = '\n'.join('{:19}{} US$/troyounce'.format( - metal + 'price', - price, - ) for metal, price in metals.items()) +####################################################### +# +# Get precious metals data and bitcoin +# -bitcoinstr = '{:{}}{} US$ # From services.packetizer.com/btc\n'.format( +metals = getjson('https://services.packetizer.com/spotprices',{'f':'json'}) +bitcoin = getjson('https://services.packetizer.com/btc',{'f':'json'}) + +metallist = ['']*len(validmetals) +for metal, price in metals.items(): + if metal in validmetals: + metalindex = validmetals.index(metal) + if validfloat(price): + if not metallist[metalindex]: + metallist[validmetals.index(metal)] = '{:19}{} US$/troyounce'.format( + metal + 'price', price) + elif verbose: + stderr.write('Got value "{}" for metal "{}" but ' + 'it is already defined\n'.format(price,metal)) + else: + stderr.write('Got invalid rate "{}" for metal "{}"\n'.format(price,metal)) + elif metal != 'date' and verbose: # Don't print a message for the "date" entry + stderr.write('Got unknown metal "{}" with value "{}"\n'.format(metal,price)) +metalstr = '\n'.join(metallist) + +if validfloat(bitcoin['usd']): + bitcoinstr = '{:{}}{} US$ # From services.packetizer.com/btc\n'.format( 'bitcoin',maxlen,bitcoin['usd']) +else: + stderr.write('Got invalid bitcoin rate "{}"\n', bitcoint['usd']) + bitcointstr='' + +####################################################### +# +# Format output and write the currency file +# +datestr = date.today().isoformat() + outstr = ( """# ISO Currency Codes {codestr} -# Currency exchange rates from Yahoo Finance (finance.yahoo.com) +# Currency exchange rates source -!message Currency exchange rates from finance.yahoo.com on {datestr} +!message Currency exchange rates from {sourcename} on {datestr} {ratestr} {bitcoinstr} # Precious metals prices from Packetizer (services.packetizer.com/spotprices) -{ozzystr} +{metalstr} -""".format(codestr=codestr, datestr=datestr, ratestr=ratestr, ozzystr=ozzystr, - bitcoinstr=bitcoinstr) +""".format(codestr=codestr, datestr=datestr, ratestr=ratestr, metalstr=metalstr, + bitcoinstr=bitcoinstr, sourcename=sourcename) ).replace('\n', linesep) try: -- 2.33.0