Fix RSA bug that I did not generate random primes for p and q

src/rsa.rs

@@ -2,6 +2,7 @@ use num_bigint::BigUint;
 use num_bigint::RandBigInt;
 use openssl::bn::BigNum;
 use openssl::bn::BigNumContext;
+use openssl::bn::MsbOption;
 use openssl::error::ErrorStack;
 
 #[derive(Clone)]
@@ -34,39 +35,52 @@ pub struct RsaPublicKey {
 }
 
 pub struct RsaPrivateKey {
-    d: BigNum,
-    n: BigNum,
+    pub d: BigNum,
+    pub n: BigNum,
+}
+
+fn generate_random_prime(bits: i32) -> Result<BigNum, ErrorStack> {
+    let mut p = BigNum::new()?;
+    let mut ctx = BigNumContext::new()?;
+    p.rand(bits, MsbOption::MAYBE_ZERO, true)?;
+    while !p.is_prime_fasttest(10, &mut ctx, true)? {
+        p.add_word(1)?;
+    }
+    Ok(p)
 }
 
 pub fn rsa_gen_keys() -> Result<(RsaPublicKey, RsaPrivateKey), ErrorStack> {
     let mut ctx = BigNumContext::new()?;
 
-    // Generate 2 random primes. We'll use small numbers to start, so you can just pick them out of a prime table. Call them "p" and "q".
-    let mut p = BigNum::new()?;
-    let mut q = BigNum::new()?;
-    p.generate_prime(256, true, None, None)?;
-    q.generate_prime(256, true, None, None)?;
+    loop {
+        // Generate 2 random primes.
+        let mut p = generate_random_prime(256)?;
+        let mut q = generate_random_prime(256)?;
 
-    // Let n be p * q. Your RSA math is modulo n.
-    let mut n = BigNum::new()?;
-    n.checked_mul(&p, &q, &mut ctx)?;
-    // This is stupid but I couldn't figure out how to clone a bignum so we do this.
-    let mut n2 = BigNum::new()?;
-    n2.checked_mul(&p, &q, &mut ctx)?;
+        // Let n be p * q. Your RSA math is modulo n.
+        let mut n = BigNum::new()?;
+        n.checked_mul(&p, &q, &mut ctx)?;
+        // This is stupid but I couldn't figure out how to clone a bignum so we do this.
+        let mut n2 = BigNum::new()?;
+        n2.checked_mul(&p, &q, &mut ctx)?;
 
-    // Let et be (p-1)*(q-1) (the "totient"). You need this value only for keygen.
-    let mut et = BigNum::new()?;
-    q.sub_word(1)?;
-    p.sub_word(1)?;
-    et.checked_mul(&p, &q, &mut ctx)?;
+        // Let et be (p-1)*(q-1) (the "totient"). You need this value only for keygen.
+        let mut et = BigNum::new()?;
+        q.sub_word(1)?;
+        p.sub_word(1)?;
+        et.checked_mul(&p, &q, &mut ctx)?;
 
-    // Let e be 3.
-    // Compute d = invmod(e, et). invmod(17, 3120) is 2753.
-    let e = BigNum::from_u32(3)?;
-    let d = invmod(&e, &et)?;
+        // Let e be 3.
+        // Compute d = invmod(e, et). invmod(17, 3120) is 2753.
+        let e = BigNum::from_u32(3)?;
+        let d = match invmod(&e, &et) {
+            Ok(i) => i,
+            Err(_) => continue,
+        };
 
-    // Your public key is [e, n]. Your private key is [d, n].
-    Ok((RsaPublicKey { e, n }, RsaPrivateKey { d, n: n2 }))
+        // Your public key is [e, n]. Your private key is [d, n].
+        return Ok((RsaPublicKey { e, n }, RsaPrivateKey { d, n: n2 }));
+    }
 }
 
 pub fn invmod(a: &BigNum, n: &BigNum) -> Result<BigNum, ErrorStack> {
@@ -109,7 +123,8 @@ pub fn invmod(a: &BigNum, n: &BigNum) -> Result<BigNum, ErrorStack> {
     let a_cloned = BigNum::from_hex_str(&a.to_hex_str()?)?;
     let n_cloned = BigNum::from_hex_str(&n.to_hex_str()?)?;
 
-    let (_, u1, _) = extended_gcd(a_cloned, n_cloned)?;
+    // if v1 == 0 there is no mod_inverse
+    let (_, u1, _v1) = extended_gcd(a_cloned, n_cloned)?;
     let r_manual = &(&(&u1 % n) + n) % n;
 
     let mut ctx = BigNumContext::new()?;
@@ -120,6 +135,7 @@ pub fn invmod(a: &BigNum, n: &BigNum) -> Result<BigNum, ErrorStack> {
 }
 
 pub fn rsa_encrypt(m: &BigNum, p: &RsaPublicKey) -> Result<BigNum, ErrorStack> {
+    assert!(m < &p.n, "message must be smaller than n");
     let mut ctx = BigNumContext::new()?;
     let mut c = BigNum::new()?;
     // To encrypt: c = m**e%n.