Fix RSA bug that I did not generate random primes for p and q

src/main.rs

@@ -62,11 +62,12 @@ fn main() {
         set5::challenge33();
         set5::challenge34();
         set5::challenge35();
-        set5::challenge36().unwrap_or_else(|| println!("[fail] challenge 36"));
-        set5::challenge37().unwrap_or_else(|| println!("[fail] challenge 37"));
     }
+    set5::challenge36().unwrap_or_else(|| println!("[fail] challenge 36"));
+    set5::challenge37().unwrap_or_else(|| println!("[fail] challenge 37"));
     set5::challenge38().unwrap_or_else(|| println!("[fail] challenge 38"));
     set5::challenge39().unwrap_or_else(|| println!("[fail] challenge 39"));
     set5::challenge40().unwrap_or_else(|| println!("[fail] challenge 40"));
     set6::challenge41().unwrap_or_else(|| println!("[fail] challenge 41"));
+    set6::challenge42().unwrap_or_else(|| println!("[fail] challenge 42"));
 }

src/rsa.rs

@@ -2,6 +2,7 @@ use num_bigint::BigUint;
 use num_bigint::RandBigInt;
 use openssl::bn::BigNum;
 use openssl::bn::BigNumContext;
+use openssl::bn::MsbOption;
 use openssl::error::ErrorStack;
 
 #[derive(Clone)]
@@ -34,39 +35,52 @@ pub struct RsaPublicKey {
 }
 
 pub struct RsaPrivateKey {
-    d: BigNum,
-    n: BigNum,
+    pub d: BigNum,
+    pub n: BigNum,
+}
+
+fn generate_random_prime(bits: i32) -> Result<BigNum, ErrorStack> {
+    let mut p = BigNum::new()?;
+    let mut ctx = BigNumContext::new()?;
+    p.rand(bits, MsbOption::MAYBE_ZERO, true)?;
+    while !p.is_prime_fasttest(10, &mut ctx, true)? {
+        p.add_word(1)?;
+    }
+    Ok(p)
 }
 
 pub fn rsa_gen_keys() -> Result<(RsaPublicKey, RsaPrivateKey), ErrorStack> {
     let mut ctx = BigNumContext::new()?;
 
-    // Generate 2 random primes. We'll use small numbers to start, so you can just pick them out of a prime table. Call them "p" and "q".
-    let mut p = BigNum::new()?;
-    let mut q = BigNum::new()?;
-    p.generate_prime(256, true, None, None)?;
-    q.generate_prime(256, true, None, None)?;
+    loop {
+        // Generate 2 random primes.
+        let mut p = generate_random_prime(256)?;
+        let mut q = generate_random_prime(256)?;
 
-    // Let n be p * q. Your RSA math is modulo n.
-    let mut n = BigNum::new()?;
-    n.checked_mul(&p, &q, &mut ctx)?;
-    // This is stupid but I couldn't figure out how to clone a bignum so we do this.
-    let mut n2 = BigNum::new()?;
-    n2.checked_mul(&p, &q, &mut ctx)?;
+        // Let n be p * q. Your RSA math is modulo n.
+        let mut n = BigNum::new()?;
+        n.checked_mul(&p, &q, &mut ctx)?;
+        // This is stupid but I couldn't figure out how to clone a bignum so we do this.
+        let mut n2 = BigNum::new()?;
+        n2.checked_mul(&p, &q, &mut ctx)?;
 
-    // Let et be (p-1)*(q-1) (the "totient"). You need this value only for keygen.
-    let mut et = BigNum::new()?;
-    q.sub_word(1)?;
-    p.sub_word(1)?;
-    et.checked_mul(&p, &q, &mut ctx)?;
+        // Let et be (p-1)*(q-1) (the "totient"). You need this value only for keygen.
+        let mut et = BigNum::new()?;
+        q.sub_word(1)?;
+        p.sub_word(1)?;
+        et.checked_mul(&p, &q, &mut ctx)?;
 
-    // Let e be 3.
-    // Compute d = invmod(e, et). invmod(17, 3120) is 2753.
-    let e = BigNum::from_u32(3)?;
-    let d = invmod(&e, &et)?;
+        // Let e be 3.
+        // Compute d = invmod(e, et). invmod(17, 3120) is 2753.
+        let e = BigNum::from_u32(3)?;
+        let d = match invmod(&e, &et) {
+            Ok(i) => i,
+            Err(_) => continue,
+        };
 
-    // Your public key is [e, n]. Your private key is [d, n].
-    Ok((RsaPublicKey { e, n }, RsaPrivateKey { d, n: n2 }))
+        // Your public key is [e, n]. Your private key is [d, n].
+        return Ok((RsaPublicKey { e, n }, RsaPrivateKey { d, n: n2 }));
+    }
 }
 
 pub fn invmod(a: &BigNum, n: &BigNum) -> Result<BigNum, ErrorStack> {
@@ -109,7 +123,8 @@ pub fn invmod(a: &BigNum, n: &BigNum) -> Result<BigNum, ErrorStack> {
     let a_cloned = BigNum::from_hex_str(&a.to_hex_str()?)?;
     let n_cloned = BigNum::from_hex_str(&n.to_hex_str()?)?;
 
-    let (_, u1, _) = extended_gcd(a_cloned, n_cloned)?;
+    // if v1 == 0 there is no mod_inverse
+    let (_, u1, _v1) = extended_gcd(a_cloned, n_cloned)?;
     let r_manual = &(&(&u1 % n) + n) % n;
 
     let mut ctx = BigNumContext::new()?;
@@ -120,6 +135,7 @@ pub fn invmod(a: &BigNum, n: &BigNum) -> Result<BigNum, ErrorStack> {
 }
 
 pub fn rsa_encrypt(m: &BigNum, p: &RsaPublicKey) -> Result<BigNum, ErrorStack> {
+    assert!(m < &p.n, "message must be smaller than n");
     let mut ctx = BigNumContext::new()?;
     let mut c = BigNum::new()?;
     // To encrypt: c = m**e%n.

src/set6.rs

@@ -16,7 +16,8 @@ pub fn challenge41() -> Option<()> {
     public_key.n.rand_range(&mut s).ok()?;
     // C' = ((S**E mod N) C) mod N
     let mut c2 = BigNum::new().ok()?;
-    c2.mod_exp(&s, &public_key.e, &public_key.n, &mut ctx).ok()?;
+    c2.mod_exp(&s, &public_key.e, &public_key.n, &mut ctx)
+        .ok()?;
     let c2 = &(&c2 * &c) % &public_key.n;
     let p2 = rsa::rsa_decrypt(&c2, &private_key).ok()?;
 
@@ -29,3 +30,15 @@ pub fn challenge41() -> Option<()> {
     println!("[okay] Challenge 41: implement unpadded message recovery oracle");
     Some(())
 }
+
+pub fn challenge42() -> Option<()> {
+    let (public_key, private_key) = rsa::rsa_gen_keys().ok()?;
+
+    let i = BigNum::from_u32(1337).ok()?;
+    let c = rsa::rsa_encrypt(&i, &public_key).ok()?;
+    let m = rsa::rsa_decrypt(&c, &private_key).ok()?;
+    assert_eq!(i, m, "rsa is broken");
+
+    println!("[xxxx] Challenge 42: Bleichenbacher's e=3 RSA Attack");
+    None
+}