Implement RSA padding which should allow me to finish challenge 42 soon

src/rsa.rs

@@ -4,6 +4,7 @@ use openssl::bn::BigNum;
 use openssl::bn::BigNumContext;
 use openssl::bn::MsbOption;
 use openssl::error::ErrorStack;
+use openssl::sha::sha256;
 
 #[derive(Clone)]
 pub struct PublicKey(pub BigUint);
@@ -54,8 +55,8 @@ pub fn rsa_gen_keys() -> Result<(RsaPublicKey, RsaPrivateKey), ErrorStack> {
 
     loop {
         // Generate 2 random primes.
-        let mut p = generate_random_prime(256)?;
-        let mut q = generate_random_prime(256)?;
+        let mut p = generate_random_prime(512)?;
+        let mut q = generate_random_prime(512)?;
 
         // Let n be p * q. Your RSA math is modulo n.
         let mut n = BigNum::new()?;
@@ -71,7 +72,7 @@ pub fn rsa_gen_keys() -> Result<(RsaPublicKey, RsaPrivateKey), ErrorStack> {
         et.checked_mul(&p, &q, &mut ctx)?;
 
         // Let e be 3.
-        // Compute d = invmod(e, et). invmod(17, 3120) is 2753.
+        // Compute d = invmod(e, et)
         let e = BigNum::from_u32(3)?;
         let d = match invmod(&e, &et) {
             Ok(i) => i,
@@ -134,16 +135,62 @@ pub fn invmod(a: &BigNum, n: &BigNum) -> Result<BigNum, ErrorStack> {
     Ok(r_manual)
 }
 
+pub fn rsa_padding_add_pkcs1(m: &BigNum, to_len: i32) -> Result<BigNum, ErrorStack> {
+    const PKCS_PADDING_SIZE: i32 = 11;
+    let from_len = m.num_bytes();
+    assert!(
+        from_len + PKCS_PADDING_SIZE <= to_len,
+        "message too long for padding"
+    );
+    let padding_str_len: usize = (to_len - 3 - from_len).try_into().unwrap();
+    let mut v = vec![0x0; 3 + padding_str_len];
+    v[0] = 0x0;
+    v[1] = 0x1;
+    for i in 2..padding_str_len + 2 {
+        v[i] = 0xff;
+    }
+    v[padding_str_len + 2] = 0x0;
+    v.append(&mut m.to_vec());
+    BigNum::from_slice(&v)
+}
+
+pub fn rsa_padding_remove_pkcs1(m: &BigNum, pad_to: i32) -> Result<BigNum, ErrorStack> {
+    // 00 || 01 || padding string || 00 || data
+    let v = m.to_vec_padded(pad_to)?;
+    let mut i = 2;
+    // first byte is zero and therefore num_bytes is 1 smaller than expected
+    assert!(m.num_bytes() + 1 == pad_to, "Padding length incorrect");
+    assert!(v[0] == 0, "PKCS1 padding incorrect");
+    assert!(v[1] == 1, "PKCS1 padding incorrect");
+    while v[i] == 0xff {
+        i += 1;
+    }
+    assert!(v[i] == 0, "PKCS1 padding incorrect");
+    BigNum::from_slice(&v[i + 1..])
+}
+
 pub fn rsa_encrypt(m: &BigNum, p: &RsaPublicKey) -> Result<BigNum, ErrorStack> {
     assert!(m < &p.n, "message must be smaller than n");
-    let mut ctx = BigNumContext::new()?;
-    let mut c = BigNum::new()?;
-    // To encrypt: c = m**e%n.
-    c.mod_exp(&m, &p.e, &p.n, &mut ctx)?;
+    let m = rsa_padding_add_pkcs1(&m, p.n.num_bytes())?;
+    let c = rsa_encrypt_unpadded(&m, p)?;
     Ok(c)
 }
 
 pub fn rsa_decrypt(c: &BigNum, p: &RsaPrivateKey) -> Result<BigNum, ErrorStack> {
+    let m = rsa_decrypt_unpadded(c, p)?;
+    let m = rsa_padding_remove_pkcs1(&m, p.n.num_bytes())?;
+    Ok(m)
+}
+
+pub fn rsa_encrypt_unpadded(m: &BigNum, p: &RsaPublicKey) -> Result<BigNum, ErrorStack> {
+    assert!(m < &p.n, "message must be smaller than n");
+    let mut ctx = BigNumContext::new()?;
+    let mut c = BigNum::new()?;
+    c.mod_exp(&m, &p.e, &p.n, &mut ctx)?;
+    Ok(c)
+}
+
+pub fn rsa_decrypt_unpadded(c: &BigNum, p: &RsaPrivateKey) -> Result<BigNum, ErrorStack> {
     let mut ctx = BigNumContext::new()?;
     let mut m = BigNum::new()?;
     // To decrypt: m = c**d%n.
@@ -162,3 +209,47 @@ pub fn rsa_decrypt_str(c: &BigNum, p: &RsaPrivateKey) -> Result<String, ErrorSta
     let m = rsa_decrypt(c, p)?;
     Ok(String::from(std::str::from_utf8(&m.to_vec()).unwrap()))
 }
+
+pub fn rsa_sign(m: &str, p: &RsaPublicKey) -> Result<BigNum, ErrorStack> {
+    let hash = sha256(m.as_bytes());
+    let m = BigNum::from_slice(&hash)?;
+    rsa_encrypt(&m, p)
+}
+
+pub fn rsa_verify(m: &str, p: &RsaPrivateKey, signature: &BigNum) -> Result<bool, ErrorStack> {
+    let hash = BigNum::from_slice(&sha256(m.as_bytes()))?;
+    let m = rsa_decrypt(signature, p)?;
+    Ok(m == hash)
+}
+
+pub fn rsa_verify_insecure(
+    m: &str,
+    p: &RsaPrivateKey,
+    signature: &BigNum,
+) -> Result<bool, ErrorStack> {
+    let hash = BigNum::from_slice(&sha256(m.as_bytes()))?;
+    const SHA256_HASH_LEN: usize = 32;
+    let pad_to = p.n.num_bytes();
+    let m = rsa_decrypt_unpadded(signature, p)?;
+    println!("{:?}", m.to_vec());
+    assert!(m.num_bytes() + 1 == pad_to, "Padding length incorrect");
+
+    // There was, 7 years ago, a common implementation flaw with RSA verifiers: they'd verify
+    // signatures by "decrypting" them (cubing them modulo the public exponent) and then "parsing"
+    // them by looking for 00h 01h ... ffh 00h ASN.1 HASH.
+    let v = m.to_vec_padded(pad_to)?;
+    println!("{:?}", v);
+    let pad_to: usize = pad_to.try_into().unwrap();
+    assert!(v[0] == 0, "PKCS1 padding incorrect");
+    assert!(v[1] == 1, "PKCS1 padding incorrect");
+    assert!(
+        v[pad_to - SHA256_HASH_LEN - 2] == 0xff,
+        "PKCS1 padding incorrect"
+    );
+    assert!(
+        v[pad_to - SHA256_HASH_LEN - 1] == 0,
+        "PKCS1 padding incorrect"
+    );
+    let sig = BigNum::from_slice(&v[pad_to - SHA256_HASH_LEN..])?;
+    Ok(sig == hash)
+}