Fix pkeyutl/rsautl empty encrypt-input/decrypt-output handling

Also fix option processing in pkeyutl to allow use of (formerly)
"out-of-order" switches that were needless implementation limitations.

Handle documented "ENGINE" form with -keyform and -peerform.

Better handling of OPENSSL_NO_ENGINE and OPENSSL_NO_RSA.

RT2018

Reviewed-by: Rich Salz <rsalz@openssl.org>
diff --git a/apps/opt.c b/apps/opt.c
index 2fbc9fe..badff26 100644
--- a/apps/opt.c
+++ b/apps/opt.c
@@ -182,8 +182,9 @@
         assert(o->name[0] != '-');
         assert(o->retval > 0);
         switch (i) {
-        case   0: case '-': case '/': case '<': case '>': case 'F': case 'M':
-        case 'L': case 'U': case 'f': case 'n': case 'p': case 's': case 'u':
+        case   0: case '-': case '/': case '<': case '>': case 'E': case 'F':
+        case 'M': case 'U': case 'f': case 'l': case 'n': case 'p': case 's':
+        case 'u':
             break;
         default:
             assert(0);
@@ -734,7 +735,7 @@
                 return -1;
             }
             break;
-        case 'L':
+        case 'l':
             if (!opt_long(arg, &lval)) {
                 BIO_printf(bio_err,
                            "%s: Invalid number \"%s\" for -%s\n",
@@ -750,9 +751,11 @@
                 return -1;
             }
             break;
-        case 'f':
+        case 'E':
         case 'F':
+        case 'f':
             if (opt_format(arg,
+                           o->valtype == 'E' ? OPT_FMT_PDE :
                            o->valtype == 'F' ? OPT_FMT_PEMDER
                            : OPT_FMT_ANY, &ival))
                 break;
@@ -823,15 +826,23 @@
     case '>':
         return "outfile";
     case 'p':
-        return "pnum";
+        return "+int";
     case 'n':
-        return "num";
+        return "int";
+    case 'l':
+        return "long";
     case 'u':
-        return "unum";
+        return "ulong";
+    case 'E':
+        return "PEM|DER|ENGINE";
     case 'F':
-        return "der/pem";
+        return "PEM|DER";
     case 'f':
         return "format";
+    case 'M':
+        return "intmax";
+    case 'U':
+        return "uintmax";
     }
     return "parm";
 }