From 321eba0f7943a7172dbfce6d470145e48ada6397 Mon Sep 17 00:00:00 2001
From: wang--ge <wang__ge@126.com>
Date: Mon, 18 Sep 2023 17:23:17 +0800
Subject: [PATCH] units 2.17 units_cur validate

---
 units_cur | 322 +++++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 245 insertions(+), 77 deletions(-)

diff --git a/units_cur b/units_cur
index 00281d8..70c8d8e 100755
--- a/units_cur
+++ b/units_cur
@@ -28,8 +28,17 @@ from __future__ import absolute_import, division, print_function
 #
 #
 
-version = '4.2'
+version = '5.0'
 
+# Version 5.0:
+#
+# Rewrite to support multiple different data sources due to disappearance
+# of the Yahoo feed.  Includes support for base currency selection. 
+#
+# Version 4.3: 20 July 2018
+#
+# Validate rate data from server
+#
 # Version 4.2: 18 April 2018
 #
 # Handle case of empty/malformed entry returned from the server
@@ -55,10 +64,17 @@ from sys import exit, stderr, stdout
 
 outfile_name = 'currency.units'
 
+# valid metals 
+
+validmetals = ['silver','gold','platinum']
+
+PRIMITIVE = '!           # Base unit, the primitive unit of currency'
+
 # This exchange rate table lists the currency ISO 4217 codes, their
 # long text names, and any fixed definitions.  If the definition is
 # empty then units_cur will query the server for a value.
 
+rate_index = 1
 currency = OrderedDict([
     ('ATS', ['austriaschilling',  '1|13.7603 euro']),
     ('BEF', ['belgiumfranc',      '1|40.3399 euro']),
@@ -83,13 +99,14 @@ currency = OrderedDict([
     ('BGN', ['bulgarialev',       '1|1.9558 euro']),
     ('BAM', ['bosniaconvertiblemark','germanymark']),
     ('KMF', ['comorosfranc',      '1|491.96775 euro']),
-    ('XOF', ['westafricanfranc',  '1|655.957 euro']),
+    ('XOF', ['westafricafranc',   '1|655.957 euro']),
     ('XPF', ['cfpfranc',          '1|119.33 euro']),
-    ('XAF', ['centralafricancfafranc','1|655.957 euro']),
+    ('XAF', ['centralafricacfafranc','1|655.957 euro']),
     ('AED', ['uaedirham','']),
     ('AFN', ['afghanafghani','']),
     ('ALL', ['albanialek','']),
     ('AMD', ['armeniadram','']),
+    ('ANG', ['antillesguilder','']),
     ('AOA', ['angolakwanza','']),
     ('ARS', ['argentinapeso','']),
     ('AUD', ['australiadollar','']),
@@ -109,7 +126,7 @@ currency = OrderedDict([
     ('BTN', ['bhutanngultrum','']),
     ('BWP', ['botswanapula','']),
     ('BYN', ['belarusruble','']),
-    ('BYR', ['oldbelarusruble','10000 BYN']),  
+    ('BYR', ['oldbelarusruble','1|10000 BYN']),  
     ('BZD', ['belizedollar','']),
     ('CAD', ['canadadollar','']),
     ('CDF', ['drcfranccongolais','']),
@@ -127,7 +144,7 @@ currency = OrderedDict([
     ('DZD', ['algeriadinar','']),
     ('EGP', ['egyptpound','']),
     ('ERN', ['eritreanakfa','']),
-    ('ETB', ['ethiopianbirr','']),
+    ('ETB', ['ethiopiabirr','']),
     ('EUR', ['euro','']),
     ('FJD', ['fijidollar','']),
     ('FKP', ['falklandislandspound','']),
@@ -164,10 +181,9 @@ currency = OrderedDict([
     ('KZT', ['kazakhstantenge','']),
     ('LAK', ['laokip','']),
     ('LBP', ['lebanonpound','']),
-    ('LKR', ['srilankanrupee','']),
+    ('LKR', ['srilankarupee','']),
     ('LRD', ['liberiadollar','']),
-    ('LTL', ['lithuanialita','']), 
-    ('LVL', ['latvialat','']),
+    ('LSL', ['lesotholoti','']),
     ('LYD', ['libyadinar','']),
     ('MAD', ['moroccodirham','']),
     ('MDL', ['moldovaleu','']),
@@ -176,13 +192,14 @@ currency = OrderedDict([
     ('MMK', ['myanmarkyat','']),
     ('MNT', ['mongoliatugrik','']),  
     ('MOP', ['macaupataca','']),
-    ('MRO', ['mauritaniaouguiya','']),
+    ('MRO', ['mauritaniaoldouguiya','1|10 MRU']),
+    ('MRU', ['mauritaniaouguiya', '']), 
     ('MUR', ['mauritiusrupee','']),
     ('MVR', ['maldiverufiyaa','']),
     ('MWK', ['malawikwacha','']),
     ('MXN', ['mexicopeso','']),
     ('MYR', ['malaysiaringgit','']),
-    ('MZN', ['mozambicanmetical','']),
+    ('MZN', ['mozambiquemetical','']),
     ('NAD', ['namibiadollar','']),
     ('NGN', ['nigerianaira','']),
     ('NIO', ['nicaraguacordobaoro','']),
@@ -212,7 +229,9 @@ currency = OrderedDict([
     ('SLL', ['sierraleoneleone','']),
     ('SOS', ['somaliaschilling','']),
     ('SRD', ['surinamedollar','']),
-    ('STD', ['saotome&principedobra','']),
+    ('SSP', ['southsudanpound','']),
+    ('STD', ['saotome&principeolddobra','']),
+    ('STN', ['saotome&principedobra','']),
     ('SVC', ['elsalvadorcolon','']),
     ('SYP', ['syriapound','']),
     ('SZL', ['swazilandlilangeni','']),
@@ -227,15 +246,15 @@ currency = OrderedDict([
     ('TZS', ['tanzaniashilling','']),
     ('UAH', ['ukrainehryvnia','']),
     ('UGX', ['ugandaschilling','']),
-    ('USD', ['unitedstatesdollar', 'US$']),
+    ('USD', ['US$', '']),
     ('UYU', ['uruguaypeso','']),
     ('UZS', ['uzbekistansum','']),
-    ('VEF', ['venezuelabolivar','']),
-    ('VEB', ['venezuelaoldbolivar', '1000 VEF']),
+    ('VEF', ['venezuelabolivarfuerte','']),
+    ('VES', ['venezuelabolivarsoberano','']),
     ('VND', ['vietnamdong','']),
     ('VUV', ['vanuatuvatu','']),
     ('WST', ['samoatala','']),
-    ('XAF', ['centralafricancfafranc','']),
+    ('XAF', ['centralafricacfafranc','']),
     ('XCD', ['eastcaribbeandollar','']),
     ('XDR', ['specialdrawingrights','']),
     ('YER', ['yemenrial','']),
@@ -244,6 +263,147 @@ currency = OrderedDict([
     ('ZWL', ['zimbabwedollar','']),
 ])
 
+def validfloat(x):
+  try:
+    float(x)
+    return True
+  except ValueError:
+    return False
+
+def addrate(verbose,form,code,rate):
+  if code not in currency.keys():
+    if (verbose):
+      stderr.write('Got unknown currency with code {}\n'.format(code))
+  else:
+    if not currency[code][rate_index]:
+      if validfloat(rate):
+        currency[code][rate_index] = form.format(rate)
+      else:
+        stderr.write('Got invalid rate "{}" for currency "{}"\n'.format(
+                                  rate, code))
+    elif verbose:
+      if currency[code][rate_index] != form.format(rate):
+        stderr.write('Got value "{}" for currency "{}" but '
+                   'it is already defined as {}\n'.format(rate, code,
+                                                  currency[code][rate_index]))
+
+def getjson(address,args=None):
+  try:
+    res = requests.get(address,args)
+    res.raise_for_status()
+    return(res.json())
+  except requests.exceptions.RequestException as e:
+    stderr.write('Error connecting to currency server:\n{}.\n'.format(e))
+    exit(1)
+        
+########################################################
+#
+#  Connect to floatrates for currency update
+#
+
+def floatrates(verbose,base,dummy):
+  webdata = getjson('https://www.floatrates.com/daily/'+base+'.json')
+  for index in webdata:
+    entry = webdata[index]
+    if 'rate' not in entry or 'code' not in entry:    # Skip empty/bad entries
+      if verbose:
+        stderr.write('Got bad entry from server: '+str(entry)+'\n')
+    else:      
+      addrate(verbose,'{} '+base,entry['code'],entry['inverseRate'])
+  currency[base][rate_index] = PRIMITIVE
+  return('FloatRates ('+base+' base)')
+
+########################################################
+#
+# Connect to European central bank site
+#
+
+def eubankrates(verbose,base,dummy):
+  if verbose and base!='EUR':
+    stderr.write('European bank uses euro for base currency.  Specified base {} ignored.\n'.format(base))
+  import xml.etree.ElementTree as ET
+  try:
+    res=requests.get('https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml')
+    res.raise_for_status()
+    data = ET.fromstring(res.content)[2][0]
+  except requests.exceptions.RequestException as e:
+    stderr.write('Error connecting to currency server:\n{}.\n'.
+                 format(e))
+    exit(1)
+  for entry in data.iter():
+    if entry.get('time'):
+      continue
+    rate = entry.get('rate')
+    code = entry.get('currency')
+    if not rate or not code:                # Skip empty/bad entries
+      if verbose:
+        stderr.write('Got bad entry from server, code {} and rate {}\n'.format(code,rate))
+    else:      
+      addrate(verbose,'1|{} euro', code, rate)
+  currency['EUR'][rate_index]=PRIMITIVE
+  return('the European Central Bank (euro base)')
+      
+########################################################
+#
+# Connect to fixer.io (requires API key)
+#
+# Free API key does not allow changing base currency
+# With free key only euro base is supported, and https is not allowed
+#
+      
+def fixer(verbose,base,key):
+  if not key:
+    stderr.write('API key required for this source\n')
+    exit(1)
+  if verbose and base!='EUR':
+    stderr.write('Fixer uses euro for base currency.  Specified base {} ignored.\n'.format(base))
+  webdata = getjson('http://data.fixer.io/api/latest', {'access_key':key})
+  if not webdata['success']:
+    stderr.write('Currency server error: '+webdata['error']['info'])
+    exit(1)
+  for code in webdata['rates']:
+    addrate(verbose,'1|{} euro', code, webdata['rates'][code])
+  currency['EUR'][rate_index] = PRIMITIVE
+  return('Fixer (euro base)')
+
+########################################################
+#
+# Connect to openexchangerates (requires API key)
+#
+# Free API key does not allow changing the base currency
+#
+         
+def openexchangerates(verbose,base,key):
+  if not key:
+    stderr.write('API key required for this source\n')
+    exit(1)
+  if verbose and base!='USD':
+    stderr.write('Open Exchange Rates uses US dollar for base currency.  Specified base {} ignored.\n'.format(base))
+  webdata = getjson('https://openexchangerates.org/api/latest.json',
+                    {'app_id':key}
+                    )
+  for code in webdata['rates']:
+    addrate(verbose,'1|{} US$', code, webdata['rates'][code])
+  currency['USD'][rate_index] = PRIMITIVE
+  return('open exchange rates (USD base)')
+
+#######################################################
+#
+# list of valid source names and corresponding functions
+#
+
+sources = {       
+  'floatrates': floatrates,
+  'eubank' : eubankrates,
+  'fixer' : fixer,
+  'openexchangerates': openexchangerates,
+}
+
+#######################################################
+#
+# Argument Processing
+#
+
 ap = ArgumentParser(
     description="Update currency information for 'units' "
     "into the specified filename or if no filename is "
@@ -271,64 +431,43 @@ ap.add_argument('-v','--verbose',
                 help='display details when fetching currency data',
 )
 
-outfile_name = ap.parse_args().output_file
-verbose = ap.parse_args().verbose
+ap.add_argument('-s','--source',choices=list(sources.keys()),
+                default='floatrates',
+                help='set currency data source',
+)
 
-try:
-  res = requests.get('http://finance.yahoo.com/webservice/v1/symbols'
-                     '/allcurrencies/quote?format=json')
-  res.raise_for_status()
-  webdata = res.json()['list']['resources']
-except requests.exceptions.RequestException as e:
-  stderr.write('Error connecting to currency server:\n{}.\n'.
-               format(e))
+ap.add_argument('-b','--base',default='USD',
+                help='set the base currency (when allowed by source).  BASE should be a 3 letter ISO currency code, e.g. USD.  The specified currency will be the primitive currency unit used by units.  Only the floatrates source supports this option.',
+)                
+
+ap.add_argument('-k','--key',default='',
+                help='set API key for sources that require it'
+)
+
+args = ap.parse_args()  
+outfile_name = args.output_file
+verbose = args.verbose
+source = args.source
+base = args.base
+apikey = args.key
+
+if base not in currency.keys():
+  stderr.write('Base currency {} is not a known currency code.\n'.format(base))
   exit(1)
-      
 
-rate_index = 1
-for data in webdata:
-  entry = data['resource']['fields']
-  if 'price' not in entry or 'symbol' not in entry:    # Skip empty/bad entries
-    if verbose:
-      stderr.write('Got bad entry from server: '+str(entry)+'\n')
-  else:      
-    rate = entry['price']
-    code = entry['symbol'][0:3]
-    if code not in currency.keys():
-      if (verbose):
-        stderr.write('Got unknown currency with code {}\n'.format(code))
-    else:
-      if not currency[code][rate_index]:
-        currency[code][rate_index] = '1|{} US$'.format(rate)
-      elif verbose:
-        stderr.write('Got value "{}" for currency "{}" but '
-                     'it is already defined\n'.format(rate, code))
+########################################################
+#
+# Fetch currency data from specified curerncy source
+#
 
-        
+sourcename = sources[source](verbose,base,apikey)
+            
 # Delete currencies where we have no rate data
-for code in currency.keys():
+for code in list(currency.keys()):
   if not currency[code][rate_index]:
     if verbose:
-      stderr.write('No data for {}'.format(code))
+      stderr.write('No data for {}\n'.format(code))
     del currency[code]
-  
-try:
-  req = requests.get('http://services.packetizer.com/spotprices/?f=json')
-  req.raise_for_status()
-  metals = req.json()
-except requests.exceptions.RequestException as e:
-  stderr.write('Error connecting to spotprices server:\n{}\n'.format(e))
-  exit(1)
-
-del metals['date']
-
-try:
-  req = requests.get('http://services.packetizer.com/btc/?f=json')
-  req.raise_for_status()
-  bitcoin = req.json()
-except requests.exceptions.RequestException as e:
-  stderr.write('Error connecting to bitcoin server:\n{}\n'.format(e))
-  exit(1)
 
 cnames = [currency[code][0] for code in currency.keys()]
 crates = [currency[code][1] for code in currency.keys()]
@@ -336,40 +475,69 @@ crates = [currency[code][1] for code in currency.keys()]
 codestr = '\n'.join('{:23}{}'.
    format(code, name) for (code,name) in zip(currency.keys(), cnames))
 
-datestr = date.today().isoformat()
-
 maxlen = max(len(name) for name in cnames) + 2
 
 ratestr = '\n'.join(
     '{:{}}{}'.format(name, maxlen, rate) for (name, rate) in zip(cnames, crates)
     )
 
-ozzystr = '\n'.join('{:19}{} US$/troyounce'.format(
-    metal + 'price',
-    price,
-    ) for metal, price in metals.items())
+#######################################################
+#
+# Get precious metals data and bitcoin
+#
 
-bitcoinstr = '{:{}}{} US$ # From services.packetizer.com/btc\n'.format(
+metals = getjson('https://services.packetizer.com/spotprices',{'f':'json'})
+bitcoin = getjson('https://services.packetizer.com/btc',{'f':'json'})
+
+metallist = ['']*len(validmetals)
+for metal, price in metals.items():
+  if metal in validmetals:
+    metalindex = validmetals.index(metal)
+    if validfloat(price):
+      if not metallist[metalindex]:
+        metallist[validmetals.index(metal)] = '{:19}{} US$/troyounce'.format(
+                                                         metal + 'price', price)
+      elif verbose:
+        stderr.write('Got value "{}" for metal "{}" but '
+                     'it is already defined\n'.format(price,metal))
+    else:
+      stderr.write('Got invalid rate "{}" for metal "{}"\n'.format(price,metal))
+  elif metal != 'date' and verbose:  # Don't print a message for the "date" entry
+    stderr.write('Got unknown metal "{}" with value "{}"\n'.format(metal,price))
+metalstr = '\n'.join(metallist)
+
+if validfloat(bitcoin['usd']):
+  bitcoinstr = '{:{}}{} US$ # From services.packetizer.com/btc\n'.format(
                 'bitcoin',maxlen,bitcoin['usd'])
+else:
+  stderr.write('Got invalid bitcoin rate "{}"\n', bitcoint['usd'])
+  bitcointstr=''
+
+#######################################################
+#
+# Format output and write the currency file
+#
 
+datestr = date.today().isoformat()
+  
 outstr = (
 """# ISO Currency Codes
 
 {codestr}
 
-# Currency exchange rates from Yahoo Finance (finance.yahoo.com)
+# Currency exchange rates source 
 
-!message Currency exchange rates from finance.yahoo.com on {datestr}
+!message Currency exchange rates from {sourcename} on {datestr}
 
 {ratestr}
 {bitcoinstr}
 
 # Precious metals prices from Packetizer (services.packetizer.com/spotprices)
 
-{ozzystr}
+{metalstr}
 
-""".format(codestr=codestr, datestr=datestr, ratestr=ratestr, ozzystr=ozzystr,
-           bitcoinstr=bitcoinstr)
+""".format(codestr=codestr, datestr=datestr, ratestr=ratestr, metalstr=metalstr,
+           bitcoinstr=bitcoinstr, sourcename=sourcename)
 ).replace('\n', linesep)
 
 try:
-- 
2.33.0

