Implement RSA padding which should allow me to finish challenge 42 soon

src/rsa.rs

@@ -4,6 +4,7 @@ use openssl::bn::BigNum;
 use openssl::bn::BigNumContext;
 use openssl::bn::MsbOption;
 use openssl::error::ErrorStack;
+use openssl::sha::sha256;
 
 #[derive(Clone)]
 pub struct PublicKey(pub BigUint);
@@ -54,8 +55,8 @@ pub fn rsa_gen_keys() -> Result<(RsaPublicKey, RsaPrivateKey), ErrorStack> {
 
     loop {
         // Generate 2 random primes.
-        let mut p = generate_random_prime(256)?;
+        let mut p = generate_random_prime(512)?;
-        let mut q = generate_random_prime(256)?;
+        let mut q = generate_random_prime(512)?;
 
         // Let n be p * q. Your RSA math is modulo n.
         let mut n = BigNum::new()?;
@@ -71,7 +72,7 @@ pub fn rsa_gen_keys() -> Result<(RsaPublicKey, RsaPrivateKey), ErrorStack> {
         et.checked_mul(&p, &q, &mut ctx)?;
 
         // Let e be 3.
-        // Compute d = invmod(e, et). invmod(17, 3120) is 2753.
+        // Compute d = invmod(e, et)
         let e = BigNum::from_u32(3)?;
         let d = match invmod(&e, &et) {
             Ok(i) => i,
@@ -134,16 +135,62 @@ pub fn invmod(a: &BigNum, n: &BigNum) -> Result<BigNum, ErrorStack> {
     Ok(r_manual)
 }
 
+pub fn rsa_padding_add_pkcs1(m: &BigNum, to_len: i32) -> Result<BigNum, ErrorStack> {
+    const PKCS_PADDING_SIZE: i32 = 11;
+    let from_len = m.num_bytes();
+    assert!(
+        from_len + PKCS_PADDING_SIZE <= to_len,
+        "message too long for padding"
+    );
+    let padding_str_len: usize = (to_len - 3 - from_len).try_into().unwrap();
+    let mut v = vec![0x0; 3 + padding_str_len];
+    v[0] = 0x0;
+    v[1] = 0x1;
+    for i in 2..padding_str_len + 2 {
+        v[i] = 0xff;
+    }
+    v[padding_str_len + 2] = 0x0;
+    v.append(&mut m.to_vec());
+    BigNum::from_slice(&v)
+}
+
+pub fn rsa_padding_remove_pkcs1(m: &BigNum, pad_to: i32) -> Result<BigNum, ErrorStack> {
+    // 00 || 01 || padding string || 00 || data
+    let v = m.to_vec_padded(pad_to)?;
+    let mut i = 2;
+    // first byte is zero and therefore num_bytes is 1 smaller than expected
+    assert!(m.num_bytes() + 1 == pad_to, "Padding length incorrect");
+    assert!(v[0] == 0, "PKCS1 padding incorrect");
+    assert!(v[1] == 1, "PKCS1 padding incorrect");
+    while v[i] == 0xff {
+        i += 1;
+    }
+    assert!(v[i] == 0, "PKCS1 padding incorrect");
+    BigNum::from_slice(&v[i + 1..])
+}
+
 pub fn rsa_encrypt(m: &BigNum, p: &RsaPublicKey) -> Result<BigNum, ErrorStack> {
     assert!(m < &p.n, "message must be smaller than n");
-    let mut ctx = BigNumContext::new()?;
+    let m = rsa_padding_add_pkcs1(&m, p.n.num_bytes())?;
-    let mut c = BigNum::new()?;
+    let c = rsa_encrypt_unpadded(&m, p)?;
-    // To encrypt: c = m**e%n.
-    c.mod_exp(&m, &p.e, &p.n, &mut ctx)?;
     Ok(c)
 }
 
 pub fn rsa_decrypt(c: &BigNum, p: &RsaPrivateKey) -> Result<BigNum, ErrorStack> {
+    let m = rsa_decrypt_unpadded(c, p)?;
+    let m = rsa_padding_remove_pkcs1(&m, p.n.num_bytes())?;
+    Ok(m)
+}
+
+pub fn rsa_encrypt_unpadded(m: &BigNum, p: &RsaPublicKey) -> Result<BigNum, ErrorStack> {
+    assert!(m < &p.n, "message must be smaller than n");
+    let mut ctx = BigNumContext::new()?;
+    let mut c = BigNum::new()?;
+    c.mod_exp(&m, &p.e, &p.n, &mut ctx)?;
+    Ok(c)
+}
+
+pub fn rsa_decrypt_unpadded(c: &BigNum, p: &RsaPrivateKey) -> Result<BigNum, ErrorStack> {
     let mut ctx = BigNumContext::new()?;
     let mut m = BigNum::new()?;
     // To decrypt: m = c**d%n.
@@ -162,3 +209,47 @@ pub fn rsa_decrypt_str(c: &BigNum, p: &RsaPrivateKey) -> Result<String, ErrorSta
     let m = rsa_decrypt(c, p)?;
     Ok(String::from(std::str::from_utf8(&m.to_vec()).unwrap()))
 }
+
+pub fn rsa_sign(m: &str, p: &RsaPublicKey) -> Result<BigNum, ErrorStack> {
+    let hash = sha256(m.as_bytes());
+    let m = BigNum::from_slice(&hash)?;
+    rsa_encrypt(&m, p)
+}
+
+pub fn rsa_verify(m: &str, p: &RsaPrivateKey, signature: &BigNum) -> Result<bool, ErrorStack> {
+    let hash = BigNum::from_slice(&sha256(m.as_bytes()))?;
+    let m = rsa_decrypt(signature, p)?;
+    Ok(m == hash)
+}
+
+pub fn rsa_verify_insecure(
+    m: &str,
+    p: &RsaPrivateKey,
+    signature: &BigNum,
+) -> Result<bool, ErrorStack> {
+    let hash = BigNum::from_slice(&sha256(m.as_bytes()))?;
+    const SHA256_HASH_LEN: usize = 32;
+    let pad_to = p.n.num_bytes();
+    let m = rsa_decrypt_unpadded(signature, p)?;
+    println!("{:?}", m.to_vec());
+    assert!(m.num_bytes() + 1 == pad_to, "Padding length incorrect");
+
+    // There was, 7 years ago, a common implementation flaw with RSA verifiers: they'd verify
+    // signatures by "decrypting" them (cubing them modulo the public exponent) and then "parsing"
+    // them by looking for 00h 01h ... ffh 00h ASN.1 HASH.
+    let v = m.to_vec_padded(pad_to)?;
+    println!("{:?}", v);
+    let pad_to: usize = pad_to.try_into().unwrap();
+    assert!(v[0] == 0, "PKCS1 padding incorrect");
+    assert!(v[1] == 1, "PKCS1 padding incorrect");
+    assert!(
+        v[pad_to - SHA256_HASH_LEN - 2] == 0xff,
+        "PKCS1 padding incorrect"
+    );
+    assert!(
+        v[pad_to - SHA256_HASH_LEN - 1] == 0,
+        "PKCS1 padding incorrect"
+    );
+    let sig = BigNum::from_slice(&v[pad_to - SHA256_HASH_LEN..])?;
+    Ok(sig == hash)
+}

src/set5.rs

@@ -543,9 +543,9 @@ pub fn challenge40() -> Option<()> {
     let (p_0, _) = rsa::rsa_gen_keys().ok()?;
     let (p_1, _) = rsa::rsa_gen_keys().ok()?;
     let (p_2, _) = rsa::rsa_gen_keys().ok()?;
-    let c_0 = rsa::rsa_encrypt(&m, &p_0).ok()?;
+    let c_0 = rsa::rsa_encrypt_unpadded(&m, &p_0).ok()?;
-    let c_1 = rsa::rsa_encrypt(&m, &p_1).ok()?;
+    let c_1 = rsa::rsa_encrypt_unpadded(&m, &p_1).ok()?;
-    let c_2 = rsa::rsa_encrypt(&m, &p_2).ok()?;
+    let c_2 = rsa::rsa_encrypt_unpadded(&m, &p_2).ok()?;
 
     // 2. Using the CRT to solve for the number represented by the three ciphertexts (which are
     //    residues mod their respective pubkeys)
@@ -579,6 +579,8 @@ pub fn challenge40() -> Option<()> {
     assert_eq!(m_cubed, result, "CRT implementation did not work");
 
     let c = cube_root(&result)?;
+    // The following line would be required if PKCS1 padding is used.
+    // let c = rsa::rsa_padding_remove_pkcs1(&c, n_0.num_bytes()).ok()?;
     assert_eq!(c, m, "cube root implementation did not work");
 
     println!("[okay] Challenge 40: implement an E=3 RSA Broadcast attack");

src/set6.rs

@@ -1,13 +1,16 @@
 use crate::rsa;
+use num_bigint::BigUint;
 use openssl::bn::BigNum;
 use openssl::bn::BigNumContext;
+use openssl::error::ErrorStack;
+use openssl::sha::sha256;
 
 pub fn challenge41() -> Option<()> {
     let (public_key, private_key) = rsa::rsa_gen_keys().ok()?;
 
     let i = BigNum::from_u32(1337).ok()?;
-    let c = rsa::rsa_encrypt(&i, &public_key).ok()?;
+    let c = rsa::rsa_encrypt_unpadded(&i, &public_key).ok()?;
-    let m = rsa::rsa_decrypt(&c, &private_key).ok()?;
+    let m = rsa::rsa_decrypt_unpadded(&c, &private_key).ok()?;
     assert_eq!(i, m, "rsa is broken");
 
     let mut ctx = BigNumContext::new().ok()?;
@@ -19,7 +22,7 @@ pub fn challenge41() -> Option<()> {
     c2.mod_exp(&s, &public_key.e, &public_key.n, &mut ctx)
         .ok()?;
     let c2 = &(&c2 * &c) % &public_key.n;
-    let p2 = rsa::rsa_decrypt(&c2, &private_key).ok()?;
+    let p2 = rsa::rsa_decrypt_unpadded(&c2, &private_key).ok()?;
 
     //      P'
     // P = ---  mod N
@@ -39,6 +42,50 @@ pub fn challenge42() -> Option<()> {
     let m = rsa::rsa_decrypt(&c, &private_key).ok()?;
     assert_eq!(i, m, "rsa is broken");
 
+    let m = "a message to verify";
+    let sig = rsa::rsa_sign(&m, &public_key).ok()?;
+    let sig_ok = rsa::rsa_verify(&m, &private_key, &sig).ok()?;
+    assert!(sig_ok, "RSA verify does not work");
+    assert!(
+        rsa::rsa_verify("other message", &private_key, &sig).ok()? == false,
+        "RSA verify does not work"
+    );
+
+    fn cube_root(n: &BigNum) -> Result<BigNum, ErrorStack> {
+        let b = BigUint::from_bytes_be(&n.to_vec());
+        let b = b.nth_root(3);
+        BigNum::from_slice(&b.to_bytes_be())
+    }
+
+    pub fn rsa_fake_sign(m: &str) -> Result<BigNum, ErrorStack> {
+        let hash = sha256(m.as_bytes());
+        let padding_str_len = 1024;
+
+        let mut v = vec![0x0; 3 + padding_str_len];
+        v[0] = 0x0;
+        v[1] = 0x1;
+        for i in 2..padding_str_len + 2 {
+            v[i] = 0x0;
+        }
+        v[padding_str_len + 1] = 0xFF;
+        v[padding_str_len + 2] = 0x0;
+        v.append(&mut hash.to_vec());
+        let sig_cubed = BigNum::from_slice(&v)?;
+        let sig = cube_root(&sig_cubed)?;
+        Ok(sig)
+    }
+
+    let i = BigNum::from_u32(1337).ok()?;
+    let c = rsa::rsa_encrypt_unpadded(&i, &public_key).ok()?;
+    let m = rsa::rsa_decrypt_unpadded(&c, &private_key).ok()?;
+    println!("i={i} c={c} m={m}");
+    assert_eq!(i, m, "rsa is broken");
+
+    let m = "hi mom";
+    let sig = rsa_fake_sign(&m).ok()?;
+    let sig_ok = rsa::rsa_verify_insecure(&m, &private_key, &sig).ok()?;
+    assert!(sig_ok, "RSA verify does not work");
+
     println!("[xxxx] Challenge 42: Bleichenbacher's e=3 RSA Attack");
-    None
+    Some(())
 }