Change the treatment of stdin and stdout to allow binary data

If the output to stdout or the input from stdin is meant to be binary,
it's deeply unsetting to get the occasional LF converted to CRLF or
the other way around.  If someone happens to forget to redirect stdin
or stdout, they will get gibberish anyway, line ending conversion will
not change that.

Therefore, let's not have dup_bio_* decide unilaterally what mode the
BIO derived from stdin and stdout, and rather let the app decide by
declaring the intended format.

Reviewed-by: Tim Hudson <tjh@openssl.org>
diff --git a/apps/apps.c b/apps/apps.c
index 0b5f9fd..d4af862 100644
--- a/apps/apps.c
+++ b/apps/apps.c
@@ -470,7 +470,7 @@
             pwdbio = BIO_push(btmp, pwdbio);
 #endif
         } else if (strcmp(arg, "stdin") == 0) {
-            pwdbio = dup_bio_in();
+            pwdbio = dup_bio_in(FORMAT_TEXT);
             if (!pwdbio) {
                 BIO_printf(bio_err, "Can't open BIO for stdin\n");
                 return NULL;
@@ -687,7 +687,7 @@
 
     if (file == NULL) {
         unbuffer(stdin);
-        cert = dup_bio_in();
+        cert = dup_bio_in(format);
     } else
         cert = bio_open_default(file, 'r', format);
     if (cert == NULL)
@@ -776,7 +776,7 @@
 #endif
     if (file == NULL && maybe_stdin) {
         unbuffer(stdin);
-        key = dup_bio_in();
+        key = dup_bio_in(format);
     } else
         key = bio_open_default(file, 'r', format);
     if (key == NULL)
@@ -839,7 +839,7 @@
 #endif
     if (file == NULL && maybe_stdin) {
         unbuffer(stdin);
-        key = dup_bio_in();
+        key = dup_bio_in(format);
     } else
         key = bio_open_default(file, 'r', format);
     if (key == NULL)
@@ -2721,16 +2721,24 @@
  * does impact behavior on some platform, such as differentiating between
  * text and binary input/output on non-Unix platforms
  */
-BIO *dup_bio_in(void)
+inline int istext(int format)
 {
-    return BIO_new_fp(stdin, BIO_NOCLOSE | BIO_FP_TEXT);
+    return (format & B_FORMAT_TEXT) == B_FORMAT_TEXT;
 }
 
-BIO *dup_bio_out(void)
+BIO *dup_bio_in(int format)
 {
-    BIO *b = BIO_new_fp(stdout, BIO_NOCLOSE | BIO_FP_TEXT);
+    return BIO_new_fp(stdin,
+                      BIO_NOCLOSE | (istext(format) ? BIO_FP_TEXT : 0));
+}
+
+BIO *dup_bio_out(int format)
+{
+    BIO *b = BIO_new_fp(stdout,
+                        BIO_NOCLOSE | (istext(format) ? BIO_FP_TEXT : 0));
 #ifdef OPENSSL_SYS_VMS
-    b = BIO_push(BIO_new(BIO_f_linebuffer()), b);
+    if (istext(format))
+        b = BIO_push(BIO_new(BIO_f_linebuffer()), b);
 #endif
     return b;
 }
@@ -2746,11 +2754,11 @@
 
     switch (mode) {
     case 'a':
-        return (format & B_FORMAT_TEXT) ? "a" : "ab";
+        return istext(format) ? "a" : "ab";
     case 'r':
-        return (format & B_FORMAT_TEXT) ? "r" : "rb";
+        return istext(format) ? "r" : "rb";
     case 'w':
-        return (format & B_FORMAT_TEXT) ? "w" : "wb";
+        return istext(format) ? "w" : "wb";
     }
     /* The assert above should make sure we never reach this point */
     return NULL;
@@ -2788,7 +2796,7 @@
 #ifdef O_TRUNC
     mode |= O_TRUNC;
 #endif
-    binmode = !(format & B_FORMAT_TEXT);
+    binmode = istext(format);
     if (binmode) {
 #ifdef O_BINARY
         mode |= O_BINARY;
@@ -2828,7 +2836,7 @@
     BIO *ret;
 
     if (filename == NULL || strcmp(filename, "-") == 0) {
-        ret = mode == 'r' ? dup_bio_in() : dup_bio_out();
+        ret = mode == 'r' ? dup_bio_in(format) : dup_bio_out(format);
         if (quiet) {
             ERR_clear_error();
             return ret;
diff --git a/apps/apps.h b/apps/apps.h
index c34d22e..0901c7d 100644
--- a/apps/apps.h
+++ b/apps/apps.h
@@ -152,8 +152,8 @@
 extern BIO *bio_in;
 extern BIO *bio_out;
 extern BIO *bio_err;
-BIO *dup_bio_in(void);
-BIO *dup_bio_out(void);
+BIO *dup_bio_in(int format);
+BIO *dup_bio_out(int format);
 BIO *bio_open_owner(const char *filename, int format, int private);
 BIO *bio_open_default(const char *filename, char mode, int format);
 BIO *bio_open_default_quiet(const char *filename, char mode, int format);
diff --git a/apps/enc.c b/apps/enc.c
index 0bdba38..fc7e14c 100644
--- a/apps/enc.c
+++ b/apps/enc.c
@@ -328,7 +328,7 @@
 
     if (infile == NULL) {
         unbuffer(stdin);
-        in = dup_bio_in();
+        in = dup_bio_in(format);
     } else
         in = bio_open_default(infile, 'r', format);
     if (in == NULL)
diff --git a/apps/engine.c b/apps/engine.c
index 91af7bf..b1c1371 100644
--- a/apps/engine.c
+++ b/apps/engine.c
@@ -319,7 +319,7 @@
     OPTION_CHOICE o;
     char *prog;
 
-    out = dup_bio_out();
+    out = dup_bio_out(FORMAT_TEXT);
     prog = opt_init(argc, argv, engine_options);
     if (!engines || !pre_cmds || !post_cmds)
         goto end;
diff --git a/apps/openssl.c b/apps/openssl.c
index 0208289..39ae64d 100644
--- a/apps/openssl.c
+++ b/apps/openssl.c
@@ -293,8 +293,8 @@
 
     /* Set up some of the environment. */
     default_config_file = make_config_name();
-    bio_in = dup_bio_in();
-    bio_out = dup_bio_out();
+    bio_in = dup_bio_in(FORMAT_TEXT);
+    bio_out = dup_bio_out(FORMAT_TEXT);
     bio_err = BIO_new_fp(stderr, BIO_NOCLOSE | BIO_FP_TEXT);
 
 #if defined( OPENSSL_SYS_VMS)
diff --git a/apps/s_client.c b/apps/s_client.c
index 819cff3..3eb495a 100644
--- a/apps/s_client.c
+++ b/apps/s_client.c
@@ -1162,9 +1162,9 @@
         if (c_quiet && !c_debug) {
             bio_c_out = BIO_new(BIO_s_null());
             if (c_msg && !bio_c_msg)
-                bio_c_msg = dup_bio_out();
+                bio_c_msg = dup_bio_out(FORMAT_TEXT);
         } else if (bio_c_out == NULL)
-            bio_c_out = dup_bio_out();
+            bio_c_out = dup_bio_out(FORMAT_TEXT);
     }
 #ifndef OPENSSL_NO_SRP
     if (!app_passwd(srppass, NULL, &srp_arg.srppassin, NULL)) {
diff --git a/apps/s_server.c b/apps/s_server.c
index e7c794c..8fe1ebe 100644
--- a/apps/s_server.c
+++ b/apps/s_server.c
@@ -1575,10 +1575,10 @@
         if (s_quiet && !s_debug) {
             bio_s_out = BIO_new(BIO_s_null());
             if (s_msg && !bio_s_msg)
-                bio_s_msg = dup_bio_out();
+                bio_s_msg = dup_bio_out(FORMAT_TEXT);
         } else {
             if (bio_s_out == NULL)
-                bio_s_out = dup_bio_out();
+                bio_s_out = dup_bio_out(FORMAT_TEXT);
         }
     }
 #if !defined(OPENSSL_NO_RSA) || !defined(OPENSSL_NO_DSA) || !defined(OPENSSL_NO_EC)