tool-openssl: pkcs8 error output on decrypt (#2883)

### Description of changes: 
Align error output of "openssl pkcs8" when handling encrypted keys.

### Testing:
Added new test to prevent regression

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license and the ISC license.

Author: justsmth

tool-openssl/pkcs8.cc

@@ -100,6 +100,45 @@ static bssl::UniquePtr<EVP_PKEY> read_private_der(BIO *in_bio,
   return bssl::UniquePtr<EVP_PKEY>(d2i_PrivateKey_bio(in_bio, nullptr));
 }
 
+// Returns 1 if PEM is encrypted, 0 if not, -1 on error
+static int is_pem_encrypted(BIO *bio) {
+    char *name_ptr = nullptr;
+    char *header_ptr = nullptr;
+    unsigned char *data_ptr = nullptr;
+    long len = 0;
+
+    // Read the PEM block
+    if (!PEM_read_bio(bio, &name_ptr, &header_ptr, &data_ptr, &len)) {
+        return -1;  // Error reading PEM
+    }
+
+    // We are responsible for freeing these
+    bssl::UniquePtr<char> name(name_ptr);
+    bssl::UniquePtr<char> header(header_ptr);
+    bssl::UniquePtr<uint8_t> data(data_ptr);
+
+    int is_encrypted = 0;
+
+    // Check if there's a header with encryption info
+    if (name && strcmp(name.get(), "ENCRYPTED PRIVATE KEY") == 0) {
+        is_encrypted = 1;
+    }
+    // Check for traditional PEM encryption (by header)
+    else if (header && header.get()[0] != '\0') {
+        EVP_CIPHER_INFO cipher;
+        if (PEM_get_EVP_CIPHER_INFO(header.get(), &cipher)) {
+            is_encrypted = (cipher.cipher != nullptr) ? 1 : 0;
+        }
+    }
+
+    // Rewind buffer so it can be parsed to obtain a private key
+    if (BIO_seek(bio, 0) >= 0) {
+      return is_encrypted;
+    }
+
+    return -1;
+}
+
 static const argument_t kArguments[] = {
     {"-help", kBooleanArgument, "Display option summary"},
     {"-in", kOptionalArgument, "Input file"},
@@ -133,7 +172,7 @@ bool pkcs8Tool(const args_list_t &args) {
   bssl::UniquePtr<EVP_PKEY> pkey;
   const EVP_CIPHER *cipher = nullptr;
   bssl::UniquePtr<PKCS8_PRIV_KEY_INFO> p8inf;
-
+  bool input_is_encrypted = false;
 
   if (!ParseOrderedKeyValueArguments(parsed_args, extra_args, args,
                                      kArguments)) {
@@ -193,6 +232,18 @@ bool pkcs8Tool(const args_list_t &args) {
   if (!in) {
     fprintf(stderr, "Cannot open input file\n");
     return false;
+  } else if (inform == "PEM") {
+    switch(is_pem_encrypted(in.get())) {
+      case 0:
+        input_is_encrypted = false;
+        break;
+      case 1:
+        input_is_encrypted = true;
+        break;
+      default:
+        fprintf(stderr, "Unable to load PEM file\n");
+        return false;
+    }
   }
   if (!validate_bio_size(in.get())) {
     return false;
@@ -218,7 +269,11 @@ bool pkcs8Tool(const args_list_t &args) {
                 in.get(), passin_arg->empty() ? nullptr : passin_arg->c_str())
                 .release());
   if (!pkey) {
-    fprintf(stderr, "Unable to load private key\n");
+    if (input_is_encrypted) {
+      fprintf(stderr, "Error decrypting key\n");
+    } else {
+      fprintf(stderr, "Unable to load private key\n");
+    }
     return false;
   }
 

tool-openssl/pkcs8_test.cc

@@ -116,6 +116,59 @@ TEST_F(PKCS8Test, PKCS8ToolEncryptionTest) {
   ASSERT_TRUE(result);
 }
 
+// Verify failure output contains "Error decrypting key"
+TEST_F(PKCS8Test, PKCS8ErrorDecryptingKey) {
+  {
+   const char* passwd = "test1234";
+   bssl::UniquePtr<BIO> pass_bio(BIO_new_file(pass_path, "wb"));
+   BIO_write(pass_bio.get(), passwd, strlen(passwd));
+   BIO_flush(pass_bio.get());
+  }
+
+  std::string passfile = std::string("file:") + pass_path;
+
+  // Phase 1: Encrypt the key
+  args_list_t args_encrypt = {
+    "-passin", "pass:''",
+    "-inform", "PEM",
+    "-in", in_path,
+    "-topk8",
+    "-v2", "aes-256-cbc",
+    "-passout", passfile.c_str(),
+    "-outform", "PEM",
+    "-out", out_path
+  };
+
+  ASSERT_TRUE(pkcs8Tool(args_encrypt));
+
+  // Phase 2: Try to decrypt with wrong password (should fail)
+  args_list_t args_verify = {
+    "-passin", "pass:''",
+    "-inform", "PEM",
+    "-in", out_path,
+    "-outform", "PEM",
+  };
+
+  // Capture stderr to verify the error message
+  testing::internal::CaptureStderr();
+  bool verify_result = pkcs8Tool(args_verify);
+  std::string captured_stderr = testing::internal::GetCapturedStderr();
+
+  ASSERT_FALSE(verify_result) << "Expected decryption to fail with wrong password";
+  EXPECT_TRUE(captured_stderr.find("Error decrypting key") != std::string::npos)
+      << "Expected 'Error decrypting key' in stderr, but got: " << captured_stderr;
+
+  // Phase 3: Decrypt with correct password (should succeed)
+  args_list_t args_decrypt = {
+    "-passin", passfile.c_str(),
+    "-inform", "PEM",
+    "-in", out_path,
+    "-outform", "PEM",
+  };
+
+  ASSERT_TRUE(pkcs8Tool(args_decrypt));
+}
+
 // Test with a direct password rather than using environment variables
 TEST_F(PKCS8Test, PKCS8ToolEnvVarPasswordTest) {
   // Phase 1: Create an unencrypted PKCS8 file first