use isxdigit and apps_tohex

Replace ad-hoc ascii->hex with isxdigit and new app_tohex.

Reviewed-by: Andy Polyakov <appro@openssl.org>

apps/apps.c

@@ -2673,6 +2673,45 @@ int app_access(const char* name, int flag)
 #endif
 }
 
+int app_hex(char c)
+{
+    switch (c) {
+    default:
+    case '0':
+        return 0;
+    case '1':
+        return 1;
+    case '2':
+        return 2;
+    case '3':
+        return 3;
+    case '4':
+          return 4;
+    case '5':
+          return 5;
+    case '6':
+          return 6;
+    case '7':
+          return 7;
+    case '8':
+          return 8;
+    case '9':
+          return 9;
+    case 'a': case 'A':
+          return 0x0A;
+    case 'b': case 'B':
+          return 0x0B;
+    case 'c': case 'C':
+          return 0x0C;
+    case 'd': case 'D':
+          return 0x0D;
+    case 'e': case 'E':
+          return 0x0E;
+    case 'f': case 'F':
+          return 0x0F;
+    }
+}
+
 /* app_isdir section */
 #ifdef _WIN32
 int app_isdir(const char *name)

apps/apps.h

@@ -544,6 +544,7 @@ void store_setup_crl_download(X509_STORE *st);
 
 # define SERIAL_RAND_BITS        64
 
+int app_hex(char);
 int app_isdir(const char *);
 int app_access(const char *, int flag);
 int raw_read_stdin(void *, int);

apps/ca.c

@@ -778,16 +778,13 @@ end_of_options:
                        i + 1, j);
             goto end;
         }
-        while (*p) {
-            if (!(((*p >= '0') && (*p <= '9')) ||
-                  ((*p >= 'A') && (*p <= 'F')) ||
-                  ((*p >= 'a') && (*p <= 'f')))) {
+        for ( ; *p; p++) {
+            if (!isxdigit(*p)) {
                 BIO_printf(bio_err,
-                           "entry %d: bad serial number characters, char pos %ld, char is '%c'\n",
-                           i + 1, (long)(p - pp[DB_serial]), *p);
+                           "entry %d: bad char 0%o '%c' in serial number\n",
+                           i + 1, *p, *p);
                 goto end;
             }
-            p++;
         }
     }
     if (verbose) {

apps/enc.c

@@ -617,16 +617,11 @@ static int set_hex(char *in, unsigned char *out, int size)
         *(in++) = '\0';
         if (j == 0)
             break;
-        if ((j >= '0') && (j <= '9'))
-            j -= '0';
-        else if ((j >= 'A') && (j <= 'F'))
-            j = j - 'A' + 10;
-        else if ((j >= 'a') && (j <= 'f'))
-            j = j - 'a' + 10;
-        else {
+        if (!isxdigit(j)) {
             BIO_printf(bio_err, "non-hex digit\n");
             return (0);
         }
+        j = (unsigned char)app_hex(j);
         if (i & 1)
             out[i / 2] |= j;
         else