/root/kaspa/kaspa-fuzz/rusty-kaspa/math/src/lehmer.rs
Line | Count | Source |
1 | | //! Variable-time modular inversion via Lehmer's extended-GCD algorithm, fully |
2 | | //! stack-allocated and generic over the operand width (little-endian base-2^64 |
3 | | //! limbs). [`LehmerInvert::lehmer_invert`] is the entry point, available on |
4 | | //! every `construct_uint!` type and on primitive `u64`/`u128` (whose native |
5 | | //! division doubles as a reference oracle in tests and fuzzing); the production |
6 | | //! user is the 3072-bit MuHash element. |
7 | | //! |
8 | | //! This is a *Euclidean* extended GCD (true quotients), not the binary |
9 | | //! Bernstein-Yang "safegcd" divstep. The distinction matters: safegcd divides by |
10 | | //! 2 each step, so its cofactors carry a `2^-62k` factor that must be paid down |
11 | | //! with a full-width modular reduction every round, whereas a Euclidean GCD keeps |
12 | | //! a single cofactor pair of clean integers that grow gradually (~`n/2` limbs on |
13 | | //! average) and need only one sign fixup at the end. |
14 | | //! |
15 | | //! Outer loop ([`invert`]): each iteration extracts a 2x2 reduction matrix from |
16 | | //! the aligned top *two* words of the two remainders (the half-GCD guess |
17 | | //! [`hgcd2`]) and applies it to the full remainders ([`apply_matrix_xy`]) and to |
18 | | //! the cofactor pair ([`apply_matrix_t`]), peeling ~62 bits per matrix (~50 |
19 | | //! matrices for a 3072-bit inverse). When the guess cannot confirm even one |
20 | | //! quotient it falls back to a single exact Euclidean division step. |
21 | | //! |
22 | | //! The guess ([`hgcd2`]) works on the top two `u64` words: a quotient of 1 is |
23 | | //! resolved by one two-word subtraction and a top-word compare, so a hardware |
24 | | //! divide is needed only for `q >= 2`. A trailing half-limb refinement keeps |
25 | | //! every matrix entry below `2^63`, so the matrix-apply accumulators never |
26 | | //! overflow and carry no per-step guard. |
27 | | //! |
28 | | //! Sign convention: track only `(t0, t1)`, the cofactor of `value`, kept |
29 | | //! non-negative and growing; flip a `swapped` flag on each Euclidean swap and |
30 | | //! recover the answer's sign from it at exit. |
31 | | //! |
32 | | //! Algorithm: Knuth, TAOCP Vol. 2, sec. 4.5.2, Algorithm L. Adapted from the |
33 | | //! MIT-licensed 256-bit implementation by Dean Little (Copyright (c) 2026 Dean |
34 | | //! Little, <https://github.com/blueshift-gg/solana-secp256k1>, `src/lehmer.rs`), |
35 | | //! generalized to arbitrary limb counts with a two-word half-GCD guess. Uses |
36 | | //! native `u128`/`i128` accumulators plus the uint type's own division |
37 | | //! ([`LehmerOps::div_rem`]) for the rare multi-limb-divisor fallback; no |
38 | | //! external dependency. |
39 | | //! |
40 | | //! The limb loops are generic over the concrete `[u64; N]` buffer types |
41 | | //! ([`LimbArray`]) rather than slices, so each width monomorphizes with |
42 | | //! compile-time trip counts and elided bounds checks, keeping the 3072-bit |
43 | | //! codegen equivalent to the pre-generalization fixed-width version (kept |
44 | | //! frozen in `benches/support/lehmer_fixed48.rs` for A/B comparison). |
45 | | |
46 | | use core::cmp::Ordering; |
47 | | |
48 | | /// Diagnostics-only per-inverse operation counters (enabled with the |
49 | | /// `lehmer-instrument` feature). Used by `examples/lehmer_stats.rs` to measure |
50 | | /// the work distribution. No effect on the production path. |
51 | | #[cfg(feature = "lehmer-instrument")] |
52 | | pub mod instrument { |
53 | | use core::sync::atomic::AtomicU64; |
54 | | pub static MATRICES: AtomicU64 = AtomicU64::new(0); |
55 | | pub static FALLBACK_STEPS: AtomicU64 = AtomicU64::new(0); |
56 | | pub static DIVIDES: AtomicU64 = AtomicU64::new(0); |
57 | | pub static APPLY_XY_LIMBS: AtomicU64 = AtomicU64::new(0); |
58 | | pub static APPLY_T_LIMBS: AtomicU64 = AtomicU64::new(0); |
59 | | pub static SWAPS: AtomicU64 = AtomicU64::new(0); |
60 | | pub static INVERSES: AtomicU64 = AtomicU64::new(0); |
61 | | pub static GCD_DIV_NORM: AtomicU64 = AtomicU64::new(0); |
62 | | } |
63 | | |
64 | | macro_rules! count { |
65 | | ($name:ident += $v:expr) => {{ |
66 | | #[cfg(feature = "lehmer-instrument")] |
67 | | instrument::$name.fetch_add($v, core::sync::atomic::Ordering::Relaxed); |
68 | | }}; |
69 | | } |
70 | | |
71 | | // Invariant checks: `debug_assert!` by default (free in release), promoted to |
72 | | // always-on `assert!` under the `strict-asserts` feature so optimized fuzz and |
73 | | // test builds verify every invariant regardless of `debug-assertions`. |
74 | | #[cfg(not(feature = "strict-asserts"))] |
75 | | macro_rules! strict_assert { |
76 | | ($($arg:tt)*) => { debug_assert!($($arg)*) }; |
77 | | } |
78 | | #[cfg(feature = "strict-asserts")] |
79 | | macro_rules! strict_assert { |
80 | | ($($arg:tt)*) => { assert!($($arg)*) }; |
81 | | } |
82 | | #[cfg(not(feature = "strict-asserts"))] |
83 | | macro_rules! strict_assert_eq { |
84 | | ($($arg:tt)*) => { debug_assert_eq!($($arg)*) }; |
85 | | } |
86 | | #[cfg(feature = "strict-asserts")] |
87 | | macro_rules! strict_assert_eq { |
88 | | ($($arg:tt)*) => { assert_eq!($($arg)*) }; |
89 | | } |
90 | | |
91 | | /// Fixed-size little-endian limb buffer, blanket-implemented for every |
92 | | /// `[u64; N]`. The inversion's working storage; keeping the concrete array |
93 | | /// type generic (rather than passing slices around) monomorphizes the limb |
94 | | /// loops per width, so trip counts and bounds stay compile-time constants. |
95 | | pub trait LimbArray: Copy { |
96 | | const LEN: usize; |
97 | | const ZERO: Self; |
98 | | fn as_slice(&self) -> &[u64]; |
99 | | fn as_mut_slice(&mut self) -> &mut [u64]; |
100 | | } |
101 | | |
102 | | impl<const N: usize> LimbArray for [u64; N] { |
103 | | const LEN: usize = N; |
104 | | const ZERO: Self = [0; N]; |
105 | | #[inline(always)] |
106 | 3.57M | fn as_slice(&self) -> &[u64] { |
107 | 3.57M | self |
108 | 3.57M | } _RNvXNtCs8KLYHAG7u19_10kaspa_math6lehmerAyj30_NtB2_9LimbArray8as_sliceCsgsKBxWKCyBU_5u3072 Line | Count | Source | 106 | 3.40M | fn as_slice(&self) -> &[u64] { | 107 | 3.40M | self | 108 | 3.40M | } |
_RNvXNtCs8KLYHAG7u19_10kaspa_math6lehmerAyj38_NtB2_9LimbArray8as_sliceCsgsKBxWKCyBU_5u3072 Line | Count | Source | 106 | 175k | fn as_slice(&self) -> &[u64] { | 107 | 175k | self | 108 | 175k | } |
Unexecuted instantiation: _RNvXININtCs8KLYHAG7u19_10kaspa_math6lehmer0KpEAypNtB5_9LimbArray8as_sliceB7_ |
109 | | #[inline(always)] |
110 | 2.82M | fn as_mut_slice(&mut self) -> &mut [u64] { |
111 | 2.82M | self |
112 | 2.82M | } _RNvXNtCs8KLYHAG7u19_10kaspa_math6lehmerAyj30_NtB2_9LimbArray12as_mut_sliceCsgsKBxWKCyBU_5u3072 Line | Count | Source | 110 | 1.36M | fn as_mut_slice(&mut self) -> &mut [u64] { | 111 | 1.36M | self | 112 | 1.36M | } |
_RNvXNtCs8KLYHAG7u19_10kaspa_math6lehmerAyj38_NtB2_9LimbArray12as_mut_sliceCsgsKBxWKCyBU_5u3072 Line | Count | Source | 110 | 1.45M | fn as_mut_slice(&mut self) -> &mut [u64] { | 111 | 1.45M | self | 112 | 1.45M | } |
Unexecuted instantiation: _RNvXININtCs8KLYHAG7u19_10kaspa_math6lehmer0KpEAypNtB5_9LimbArray12as_mut_sliceB7_ |
113 | | } |
114 | | |
115 | | /// Extra limbs every cofactor buffer ([`LehmerOps::Cofactor`]) carries beyond |
116 | | /// the operand width, absorbing carry-out during a matrix application and the |
117 | | /// multi-limb fallback. Compile-time checked per width in [`invert`]. |
118 | | pub const COFACTOR_HEADROOM: usize = 8; |
119 | | |
120 | | /// What Lehmer inversion needs from a uint type beyond plain limb arithmetic: |
121 | | /// its limb representation and an exact multi-limb division (the rare fallback |
122 | | /// when the two-word guess cannot confirm a quotient, and the defensive branch |
123 | | /// of the final reduction). |
124 | | /// |
125 | | /// Implemented by `construct_uint!` for every generated type, and manually for |
126 | | /// primitive `u64`/`u128`. |
127 | | pub trait LehmerOps: Copy { |
128 | | /// The operand representation, `[u64; N]`. |
129 | | type Limbs: LimbArray; |
130 | | /// The cofactor working buffer, `[u64; N + COFACTOR_HEADROOM]`. The |
131 | | /// cofactor of `value` is bounded in magnitude by the modulus (<= `N` |
132 | | /// limbs); the extra headroom absorbs carry-out during a matrix |
133 | | /// application and the multi-limb fallback. Supplied per-impl because |
134 | | /// stable Rust cannot spell the sum generically. |
135 | | type Cofactor: LimbArray; |
136 | | fn from_limbs(limbs: Self::Limbs) -> Self; |
137 | | fn into_limbs(self) -> Self::Limbs; |
138 | | /// `(self / other, self % other)`. |
139 | | fn div_rem(self, other: Self) -> (Self, Self); |
140 | | } |
141 | | |
142 | | impl LehmerOps for u64 { |
143 | | type Limbs = [u64; 1]; |
144 | | type Cofactor = [u64; 1 + COFACTOR_HEADROOM]; |
145 | | #[inline] |
146 | 0 | fn from_limbs(limbs: Self::Limbs) -> Self { |
147 | 0 | limbs[0] |
148 | 0 | } |
149 | | #[inline] |
150 | 0 | fn into_limbs(self) -> Self::Limbs { |
151 | 0 | [self] |
152 | 0 | } |
153 | | #[inline] |
154 | 0 | fn div_rem(self, other: Self) -> (Self, Self) { |
155 | 0 | (self / other, self % other) |
156 | 0 | } |
157 | | } |
158 | | |
159 | | impl LehmerOps for u128 { |
160 | | type Limbs = [u64; 2]; |
161 | | type Cofactor = [u64; 2 + COFACTOR_HEADROOM]; |
162 | | #[inline] |
163 | 0 | fn from_limbs(limbs: Self::Limbs) -> Self { |
164 | 0 | ((limbs[1] as u128) << 64) | (limbs[0] as u128) |
165 | 0 | } |
166 | | #[inline] |
167 | 0 | fn into_limbs(self) -> Self::Limbs { |
168 | 0 | [self as u64, (self >> 64) as u64] |
169 | 0 | } |
170 | | #[inline] |
171 | 0 | fn div_rem(self, other: Self) -> (Self, Self) { |
172 | 0 | (self / other, self % other) |
173 | 0 | } |
174 | | } |
175 | | |
176 | | /// Modular inversion as a method on any [`LehmerOps`] type. |
177 | | pub trait LehmerInvert: Sized { |
178 | | /// The multiplicative inverse of `self` modulo `modulus`, in `[0, modulus)`. |
179 | | /// |
180 | | /// Total over all inputs: `self` is reduced first when `self >= modulus`, |
181 | | /// and `None` is returned when no inverse exists (`gcd != 1`, which covers |
182 | | /// a zero residue, or `modulus < 2`). |
183 | | fn lehmer_invert(self, modulus: Self) -> Option<Self>; |
184 | | } |
185 | | |
186 | | impl<U: LehmerOps> LehmerInvert for U { |
187 | 0 | fn lehmer_invert(self, modulus: Self) -> Option<Self> { |
188 | 0 | let m = modulus.into_limbs(); |
189 | 0 | let ms = m.as_slice(); |
190 | 0 | if ms[0] < 2 && ms[1..].iter().all(|&w| w == 0) { |
191 | 0 | return None; |
192 | 0 | } |
193 | 0 | let mut value = self.into_limbs(); |
194 | 0 | if cmp_n(value.as_slice(), ms) != Ordering::Less { |
195 | 0 | value = self.div_rem(modulus).1.into_limbs(); |
196 | 0 | } |
197 | 0 | let mut out = U::Limbs::ZERO; |
198 | 0 | invert::<U>(value, m, &mut out).then(|| U::from_limbs(out)) |
199 | 0 | } |
200 | | } |
201 | | |
202 | | // Matrix entries from `hgcd2` are below `2^63` by construction (the half-limb |
203 | | // refinement discards the least significant half limb), so the `i128`/`u128` |
204 | | // accumulators in the matrix applications cannot overflow |
205 | | // (`a*x + b*y + carry < 2^128` with `a, b < 2^63` and `x, y < 2^64`). |
206 | | |
207 | | /// Compute the multiplicative inverse of `value` modulo `modulus` (an odd |
208 | | /// modulus, e.g. the MuHash prime), via Lehmer's extended GCD. `value`, |
209 | | /// `modulus` and `out` are little-endian base-2^64 limb arrays, and `value` |
210 | | /// must already be reduced (`value < modulus`). |
211 | | /// [`LehmerInvert::lehmer_invert`] is the totalized wrapper without the |
212 | | /// reduction precondition. |
213 | | /// |
214 | | /// Returns `true` and writes the inverse into `out` when |
215 | | /// `gcd(value, modulus) == 1`; returns `false` (leaving `out` unspecified) |
216 | | /// otherwise. A zero `value` yields `false`. |
217 | 21.7k | pub fn invert<U: LehmerOps>(value: U::Limbs, modulus: U::Limbs, out: &mut U::Limbs) -> bool { |
218 | | const { |
219 | | assert!(U::Cofactor::LEN >= U::Limbs::LEN + COFACTOR_HEADROOM, "cofactor buffer lacks the required headroom"); |
220 | | } |
221 | 21.7k | count!(INVERSES += 1); |
222 | 21.7k | if is_zero(value.as_slice()) { |
223 | 0 | core::hint::cold_path(); |
224 | 0 | return false; |
225 | 21.7k | } |
226 | 21.7k | strict_assert!(cmp_n(value.as_slice(), modulus.as_slice()).is_lt(), "invert requires value < modulus"); |
227 | | |
228 | | // Euclidean state on (x, y) with x >= y, plus the cofactor of `value`. |
229 | | // Invariant: x ≡ -t0·value (mod m) and y ≡ t1·value (mod m) when |
230 | | // `swapped == false`, and the roles flip with each swap. `x`, `y` are owned |
231 | | // working buffers (they are reduced in place). |
232 | 21.7k | let mut x: U::Limbs = modulus; |
233 | 21.7k | let mut y: U::Limbs = value; |
234 | 21.7k | let mut x_len = top_len(x.as_slice()); |
235 | 21.7k | let mut y_len = top_len(y.as_slice()); |
236 | 21.7k | let mut t0 = U::Cofactor::ZERO; |
237 | 21.7k | let mut t1 = from_word::<U::Cofactor>(1); |
238 | 21.7k | let mut t0_len = 1usize; // 0 has conventional length 1 |
239 | 21.7k | let mut t1_len = 1usize; |
240 | 21.7k | let mut swapped = false; |
241 | | |
242 | | // Multi-limb phase: run until y collapses to a single limb, then finish |
243 | | // with a u64 extended-Euclidean tail. |
244 | 761k | while y_len > 1 { |
245 | 739k | let (x_hi, y_hi) = highest_two_words_normalized(x.as_slice(), y.as_slice(), x_len); |
246 | 739k | let guess = hgcd2((x_hi >> 64) as u64, x_hi as u64, (y_hi >> 64) as u64, y_hi as u64); |
247 | | |
248 | 739k | if let Some((a, b, c, d)) = guess { |
249 | 652k | count!(MATRICES += 1); |
250 | 652k | (x_len, y_len) = apply_matrix_xy(&mut x, &mut y, x_len, a, b, c, d); |
251 | 652k | apply_matrix_t(&mut t0, &mut t1, &mut t0_len, &mut t1_len, a, b, c, d); |
252 | | |
253 | | // A partial guess can leave x < y; restore x >= y, toggling the sign. |
254 | 652k | if x_len < y_len || (x_len == y_len && cmp_prefix(x.as_slice(), y.as_slice(), x_len).is_lt()) { |
255 | 327k | core::mem::swap(&mut x, &mut y); |
256 | 327k | core::mem::swap(&mut x_len, &mut y_len); |
257 | 327k | core::mem::swap(&mut t0, &mut t1); |
258 | 327k | core::mem::swap(&mut t0_len, &mut t1_len); |
259 | 327k | swapped = !swapped; |
260 | 327k | } |
261 | 87.3k | } else { |
262 | 87.3k | // The guess almost always confirms a quotient, so this exact-division |
263 | 87.3k | // path is cold; keep it out of the hot loop body. |
264 | 87.3k | core::hint::cold_path(); |
265 | 87.3k | count!(FALLBACK_STEPS += 1); |
266 | 87.3k | // Guess could not confirm a quotient: one exact Euclidean step. |
267 | 87.3k | let (q, r) = div_rem_step::<U>(&x, &y); |
268 | 87.3k | t_add_qmul(&mut t0, &mut t0_len, q.as_slice(), &t1, t1_len); |
269 | 87.3k | x = y; |
270 | 87.3k | x_len = y_len; |
271 | 87.3k | y = r; |
272 | 87.3k | y_len = top_len(y.as_slice()); |
273 | 87.3k | core::mem::swap(&mut t0, &mut t1); |
274 | 87.3k | core::mem::swap(&mut t0_len, &mut t1_len); |
275 | 87.3k | swapped = !swapped; |
276 | 87.3k | } |
277 | | } |
278 | | |
279 | | // Single-limb tail. `y` is now <= 1 limb; one u64 extended Euclidean step |
280 | | // replaces the remaining ~tens of `t_add_qmul` iterations. |
281 | 21.7k | if y_len == 1 && y.as_slice()[0] != 0 { |
282 | 973k | if x.as_slice()[1..].iter().any(|&w| w != 0) {_RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer6invertNtB6_8Uint3072E0CsgsKBxWKCyBU_5u3072 Line | Count | Source | 282 | 973k | if x.as_slice()[1..].iter().any(|&w| w != 0) { |
Unexecuted instantiation: _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer6invertpE0B6_ |
283 | 1.03k | let (q, r) = div_rem_step::<U>(&x, &y); |
284 | 1.03k | t_add_qmul(&mut t0, &mut t0_len, q.as_slice(), &t1, t1_len); |
285 | 1.03k | x = from_word(y.as_slice()[0]); |
286 | 1.03k | y = r; |
287 | 1.03k | core::mem::swap(&mut t0, &mut t1); |
288 | 1.03k | core::mem::swap(&mut t0_len, &mut t1_len); |
289 | 1.03k | swapped = !swapped; |
290 | 20.6k | } |
291 | | |
292 | 21.7k | let (g, cx, cy) = gcd_ext_u64(x.as_slice()[0], y.as_slice()[0]); |
293 | | |
294 | | // Fold the sign of the chosen cofactor into `swapped`, then combine |
295 | | // magnitudes: new_t0 = |cx|·t0 + |cy|·t1. |
296 | 21.7k | if cx < 0 || (cx == 0 && cy > 0) { |
297 | 10.8k | swapped = !swapped; |
298 | 10.9k | } |
299 | 21.7k | let mut new_t0 = U::Cofactor::ZERO; |
300 | 21.7k | let mut new_t0_len = 1usize; |
301 | 21.7k | add_mul_word(&mut new_t0, &mut new_t0_len, cx.unsigned_abs(), &t0, t0_len); |
302 | 21.7k | add_mul_word(&mut new_t0, &mut new_t0_len, cy.unsigned_abs(), &t1, t1_len); |
303 | 21.7k | t0 = new_t0; |
304 | | |
305 | 21.7k | x = from_word(g); |
306 | 21.7k | x_len = if g == 0 { 0 } else { 1 }; |
307 | 0 | } |
308 | | |
309 | | // For an odd prime modulus and `value` in [1, m), the gcd is 1. |
310 | 21.7k | if !(x_len == 1 && x.as_slice()[0] == 1) { |
311 | 0 | return false; |
312 | 21.7k | } |
313 | | |
314 | | // x = 1 = ±t0·value (mod m). With `swapped`, the inverse is +t0; otherwise |
315 | | // it is -t0 ≡ m - t0. Reduce t0 into [0, m) first. |
316 | 21.7k | let mut inv = reduce_mod::<U>(&t0, &modulus); |
317 | 21.7k | if !swapped && !is_zero(inv.as_slice()) { |
318 | 10.9k | inv = sub_n(&modulus, &inv); |
319 | 10.9k | } |
320 | 21.7k | strict_assert!( |
321 | 21.7k | !is_zero(inv.as_slice()) && cmp_n(inv.as_slice(), modulus.as_slice()).is_lt(), |
322 | | "invert: result outside (0, modulus)" |
323 | | ); |
324 | 21.7k | *out = inv; |
325 | 21.7k | true |
326 | 21.7k | } _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer6invertNtB4_8Uint3072ECsgsKBxWKCyBU_5u3072 Line | Count | Source | 217 | 21.7k | pub fn invert<U: LehmerOps>(value: U::Limbs, modulus: U::Limbs, out: &mut U::Limbs) -> bool { | 218 | | const { | 219 | | assert!(U::Cofactor::LEN >= U::Limbs::LEN + COFACTOR_HEADROOM, "cofactor buffer lacks the required headroom"); | 220 | | } | 221 | 21.7k | count!(INVERSES += 1); | 222 | 21.7k | if is_zero(value.as_slice()) { | 223 | 0 | core::hint::cold_path(); | 224 | 0 | return false; | 225 | 21.7k | } | 226 | 21.7k | strict_assert!(cmp_n(value.as_slice(), modulus.as_slice()).is_lt(), "invert requires value < modulus"); | 227 | | | 228 | | // Euclidean state on (x, y) with x >= y, plus the cofactor of `value`. | 229 | | // Invariant: x ≡ -t0·value (mod m) and y ≡ t1·value (mod m) when | 230 | | // `swapped == false`, and the roles flip with each swap. `x`, `y` are owned | 231 | | // working buffers (they are reduced in place). | 232 | 21.7k | let mut x: U::Limbs = modulus; | 233 | 21.7k | let mut y: U::Limbs = value; | 234 | 21.7k | let mut x_len = top_len(x.as_slice()); | 235 | 21.7k | let mut y_len = top_len(y.as_slice()); | 236 | 21.7k | let mut t0 = U::Cofactor::ZERO; | 237 | 21.7k | let mut t1 = from_word::<U::Cofactor>(1); | 238 | 21.7k | let mut t0_len = 1usize; // 0 has conventional length 1 | 239 | 21.7k | let mut t1_len = 1usize; | 240 | 21.7k | let mut swapped = false; | 241 | | | 242 | | // Multi-limb phase: run until y collapses to a single limb, then finish | 243 | | // with a u64 extended-Euclidean tail. | 244 | 761k | while y_len > 1 { | 245 | 739k | let (x_hi, y_hi) = highest_two_words_normalized(x.as_slice(), y.as_slice(), x_len); | 246 | 739k | let guess = hgcd2((x_hi >> 64) as u64, x_hi as u64, (y_hi >> 64) as u64, y_hi as u64); | 247 | | | 248 | 739k | if let Some((a, b, c, d)) = guess { | 249 | 652k | count!(MATRICES += 1); | 250 | 652k | (x_len, y_len) = apply_matrix_xy(&mut x, &mut y, x_len, a, b, c, d); | 251 | 652k | apply_matrix_t(&mut t0, &mut t1, &mut t0_len, &mut t1_len, a, b, c, d); | 252 | | | 253 | | // A partial guess can leave x < y; restore x >= y, toggling the sign. | 254 | 652k | if x_len < y_len || (x_len == y_len && cmp_prefix(x.as_slice(), y.as_slice(), x_len).is_lt()) { | 255 | 327k | core::mem::swap(&mut x, &mut y); | 256 | 327k | core::mem::swap(&mut x_len, &mut y_len); | 257 | 327k | core::mem::swap(&mut t0, &mut t1); | 258 | 327k | core::mem::swap(&mut t0_len, &mut t1_len); | 259 | 327k | swapped = !swapped; | 260 | 327k | } | 261 | 87.3k | } else { | 262 | 87.3k | // The guess almost always confirms a quotient, so this exact-division | 263 | 87.3k | // path is cold; keep it out of the hot loop body. | 264 | 87.3k | core::hint::cold_path(); | 265 | 87.3k | count!(FALLBACK_STEPS += 1); | 266 | 87.3k | // Guess could not confirm a quotient: one exact Euclidean step. | 267 | 87.3k | let (q, r) = div_rem_step::<U>(&x, &y); | 268 | 87.3k | t_add_qmul(&mut t0, &mut t0_len, q.as_slice(), &t1, t1_len); | 269 | 87.3k | x = y; | 270 | 87.3k | x_len = y_len; | 271 | 87.3k | y = r; | 272 | 87.3k | y_len = top_len(y.as_slice()); | 273 | 87.3k | core::mem::swap(&mut t0, &mut t1); | 274 | 87.3k | core::mem::swap(&mut t0_len, &mut t1_len); | 275 | 87.3k | swapped = !swapped; | 276 | 87.3k | } | 277 | | } | 278 | | | 279 | | // Single-limb tail. `y` is now <= 1 limb; one u64 extended Euclidean step | 280 | | // replaces the remaining ~tens of `t_add_qmul` iterations. | 281 | 21.7k | if y_len == 1 && y.as_slice()[0] != 0 { | 282 | 21.7k | if x.as_slice()[1..].iter().any(|&w| w != 0) { | 283 | 1.03k | let (q, r) = div_rem_step::<U>(&x, &y); | 284 | 1.03k | t_add_qmul(&mut t0, &mut t0_len, q.as_slice(), &t1, t1_len); | 285 | 1.03k | x = from_word(y.as_slice()[0]); | 286 | 1.03k | y = r; | 287 | 1.03k | core::mem::swap(&mut t0, &mut t1); | 288 | 1.03k | core::mem::swap(&mut t0_len, &mut t1_len); | 289 | 1.03k | swapped = !swapped; | 290 | 20.6k | } | 291 | | | 292 | 21.7k | let (g, cx, cy) = gcd_ext_u64(x.as_slice()[0], y.as_slice()[0]); | 293 | | | 294 | | // Fold the sign of the chosen cofactor into `swapped`, then combine | 295 | | // magnitudes: new_t0 = |cx|·t0 + |cy|·t1. | 296 | 21.7k | if cx < 0 || (cx == 0 && cy > 0) { | 297 | 10.8k | swapped = !swapped; | 298 | 10.9k | } | 299 | 21.7k | let mut new_t0 = U::Cofactor::ZERO; | 300 | 21.7k | let mut new_t0_len = 1usize; | 301 | 21.7k | add_mul_word(&mut new_t0, &mut new_t0_len, cx.unsigned_abs(), &t0, t0_len); | 302 | 21.7k | add_mul_word(&mut new_t0, &mut new_t0_len, cy.unsigned_abs(), &t1, t1_len); | 303 | 21.7k | t0 = new_t0; | 304 | | | 305 | 21.7k | x = from_word(g); | 306 | 21.7k | x_len = if g == 0 { 0 } else { 1 }; | 307 | 0 | } | 308 | | | 309 | | // For an odd prime modulus and `value` in [1, m), the gcd is 1. | 310 | 21.7k | if !(x_len == 1 && x.as_slice()[0] == 1) { | 311 | 0 | return false; | 312 | 21.7k | } | 313 | | | 314 | | // x = 1 = ±t0·value (mod m). With `swapped`, the inverse is +t0; otherwise | 315 | | // it is -t0 ≡ m - t0. Reduce t0 into [0, m) first. | 316 | 21.7k | let mut inv = reduce_mod::<U>(&t0, &modulus); | 317 | 21.7k | if !swapped && !is_zero(inv.as_slice()) { | 318 | 10.9k | inv = sub_n(&modulus, &inv); | 319 | 10.9k | } | 320 | 21.7k | strict_assert!( | 321 | 21.7k | !is_zero(inv.as_slice()) && cmp_n(inv.as_slice(), modulus.as_slice()).is_lt(), | 322 | | "invert: result outside (0, modulus)" | 323 | | ); | 324 | 21.7k | *out = inv; | 325 | 21.7k | true | 326 | 21.7k | } |
Unexecuted instantiation: _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer6invertpEB4_ |
327 | | |
328 | | /// Two-word subtract `(h1·2^64 + l1) - (h2·2^64 + l2)`, returning `(hi, lo)`. |
329 | | /// Callers guarantee the minuend is the larger, so no underflow off the top. |
330 | | #[inline(always)] |
331 | 12.1M | fn sub2(h1: u64, l1: u64, h2: u64, l2: u64) -> (u64, u64) { |
332 | 12.1M | let (lo, borrow) = l1.overflowing_sub(l2); |
333 | 12.1M | (h1.wrapping_sub(h2).wrapping_sub(borrow as u64), lo) |
334 | 12.1M | } |
335 | | |
336 | | /// Fold the next half limb up into a fresh full word, `(hi << 32) | (lo >> 32)`, |
337 | | /// for the handoff from the main hgcd2 loop (working on top *words*) to the |
338 | | /// half-limb refinement loop. `hi < 2^32` here, so the result fits a `u64`. |
339 | | #[inline(always)] |
340 | 1.25M | fn fold_half(hi: u64, lo: u64) -> u64 { |
341 | 1.25M | (hi << 32).wrapping_add(lo >> 32) |
342 | 1.25M | } |
343 | | |
344 | | /// The half-GCD guess: from the aligned top *two* words of the remainders |
345 | | /// (`(ah, al) >=~ (bh, bl)`), run Euclidean steps on the 128-bit approximation |
346 | | /// and return the 2x2 reduction matrix in this module's `(a, b, c, d)` |
347 | | /// convention, `(x', y') = (a·x - b·y, d·y - c·x)`. Returns `None` when no |
348 | | /// quotient can be confirmed (the caller then takes one exact division step). |
349 | | /// |
350 | | /// A quotient of 1 is resolved by one two-word subtraction and a top-word |
351 | | /// compare, so [`gcd_div`] (the hardware divide) is reached only for `q >= 2`. |
352 | | /// The trailing half-limb refinement (the `HALF_LIMIT_2` loop) keeps every |
353 | | /// matrix entry below `2^63`, which is why the `apply_matrix_*` accumulators |
354 | | /// need no per-step overflow guard. All matrix arithmetic is `wrapping_*` |
355 | | /// (entries are `< 2^63`, so it never actually wraps) to avoid the workspace |
356 | | /// `overflow-checks = true` panic pads. |
357 | | #[inline] |
358 | 739k | fn hgcd2(mut ah: u64, mut al: u64, mut bh: u64, mut bl: u64) -> Option<(u64, u64, u64, u64)> { |
359 | | // Need at least two leading bits of headroom in both top words. The guess |
360 | | // almost always confirms a quotient, so every `return None` here is cold. |
361 | 739k | if ah < 2 || bh < 2 { |
362 | 46.2k | core::hint::cold_path(); |
363 | 46.2k | return None; |
364 | 693k | } |
365 | | |
366 | | // Reduction matrix M = (m00 m01; m10 m11), accumulated by the same column |
367 | | // updates as the full-width apply; returned remapped to (a,b,c,d). |
368 | | let (mut m00, mut m01, mut m10, mut m11): (u64, u64, u64, u64); |
369 | 693k | if ah > bh || (ah == bh && al > bl) { |
370 | 681k | (ah, al) = sub2(ah, al, bh, bl); // a -= b |
371 | 681k | if ah < 2 { |
372 | 29.1k | return None; |
373 | 652k | } |
374 | 652k | (m01, m10) = (1, 0); |
375 | | } else { |
376 | 11.9k | (bh, bl) = sub2(bh, bl, ah, al); // b -= a |
377 | 11.9k | if bh < 2 { |
378 | 11.9k | return None; |
379 | 0 | } |
380 | 0 | (m01, m10) = (0, 1); |
381 | | } |
382 | 652k | (m00, m11) = (1, 1); |
383 | | |
384 | | const HALF: u32 = 32; |
385 | | const HALF_LIMIT_1: u64 = 1 << HALF; |
386 | | |
387 | 652k | let mut subtract_a = ah < bh; |
388 | 652k | let mut subtract_a1 = false; |
389 | 652k | let mut done = false; |
390 | | loop { |
391 | 6.52M | if subtract_a { |
392 | 624k | subtract_a = false; |
393 | 624k | } else { |
394 | 5.89M | if ah == bh { |
395 | 3.17k | done = true; |
396 | 3.17k | break; |
397 | 5.89M | } |
398 | 5.89M | if ah < HALF_LIMIT_1 { |
399 | | // Top words collapsed to a half limb: fold in the next half from |
400 | | // the low words and finish in the refinement loop below. |
401 | 314k | ah = fold_half(ah, al); |
402 | 314k | bh = fold_half(bh, bl); |
403 | 314k | break; |
404 | 5.57M | } |
405 | | // Subtract a -= q·b (affects the second column of M). |
406 | 5.57M | (ah, al) = sub2(ah, al, bh, bl); |
407 | 5.57M | if ah < 2 { |
408 | 3.75k | done = true; |
409 | 3.75k | break; |
410 | 5.57M | } |
411 | 5.57M | if ah <= bh { |
412 | 2.33M | m01 = m01.wrapping_add(m00); // q = 1: no divide |
413 | 2.33M | m11 = m11.wrapping_add(m10); |
414 | 2.33M | } else { |
415 | 3.23M | let n = ((ah as u128) << 64) | al as u128; |
416 | 3.23M | let d = ((bh as u128) << 64) | bl as u128; |
417 | 3.23M | let (mut q, r) = gcd_div(n, d); |
418 | 3.23M | ah = (r >> 64) as u64; |
419 | 3.23M | al = r as u64; |
420 | 3.23M | if ah < 2 { |
421 | 8.03k | m01 = m01.wrapping_add(q.wrapping_mul(m00)); // q correct, a too small |
422 | 8.03k | m11 = m11.wrapping_add(q.wrapping_mul(m10)); |
423 | 8.03k | done = true; |
424 | 8.03k | break; |
425 | 3.22M | } |
426 | 3.22M | q = q.wrapping_add(1); // one subtraction already taken above |
427 | 3.22M | m01 = m01.wrapping_add(q.wrapping_mul(m00)); |
428 | 3.22M | m11 = m11.wrapping_add(q.wrapping_mul(m10)); |
429 | | } |
430 | | } |
431 | 6.19M | if ah == bh { |
432 | 2.41k | done = true; |
433 | 2.41k | break; |
434 | 6.18M | } |
435 | 6.18M | if bh < HALF_LIMIT_1 { |
436 | 311k | ah = fold_half(ah, al); |
437 | 311k | bh = fold_half(bh, bl); |
438 | 311k | subtract_a1 = true; |
439 | 311k | break; |
440 | 5.87M | } |
441 | | // Subtract b -= q·a (affects the first column of M). |
442 | 5.87M | (bh, bl) = sub2(bh, bl, ah, al); |
443 | 5.87M | if bh < 2 { |
444 | 2.67k | done = true; |
445 | 2.67k | break; |
446 | 5.87M | } |
447 | 5.87M | if bh <= ah { |
448 | 2.21M | m00 = m00.wrapping_add(m01); // q = 1: no divide |
449 | 2.21M | m10 = m10.wrapping_add(m11); |
450 | 2.21M | } else { |
451 | 3.65M | let n = ((bh as u128) << 64) | bl as u128; |
452 | 3.65M | let d = ((ah as u128) << 64) | al as u128; |
453 | 3.65M | let (mut q, r) = gcd_div(n, d); |
454 | 3.65M | bh = (r >> 64) as u64; |
455 | 3.65M | bl = r as u64; |
456 | 3.65M | if bh < 2 { |
457 | 6.21k | m00 = m00.wrapping_add(q.wrapping_mul(m01)); |
458 | 6.21k | m10 = m10.wrapping_add(q.wrapping_mul(m11)); |
459 | 6.21k | done = true; |
460 | 6.21k | break; |
461 | 3.65M | } |
462 | 3.65M | q = q.wrapping_add(1); |
463 | 3.65M | m00 = m00.wrapping_add(q.wrapping_mul(m01)); |
464 | 3.65M | m10 = m10.wrapping_add(q.wrapping_mul(m11)); |
465 | | } |
466 | | } |
467 | | |
468 | | // Half-limb refinement: peel a bit more (single-word divides on the folded |
469 | | // top words) until |a - b| fits in one limb + 1 bit. This is what keeps the |
470 | | // matrix entries below 2^63, discarding the least significant half limb. |
471 | 652k | if !done { |
472 | | const HALF_LIMIT_2: u64 = 1 << (HALF + 1); |
473 | | loop { |
474 | 5.55M | if subtract_a1 { |
475 | 311k | subtract_a1 = false; |
476 | 311k | } else { |
477 | 5.24M | ah = ah.wrapping_sub(bh); |
478 | 5.24M | if ah < HALF_LIMIT_2 { |
479 | 143k | break; |
480 | 5.09M | } |
481 | 5.09M | if ah <= bh { |
482 | 2.12M | m01 = m01.wrapping_add(m00); |
483 | 2.12M | m11 = m11.wrapping_add(m10); |
484 | 2.12M | } else { |
485 | 2.97M | let q = ah / bh; |
486 | 2.97M | ah %= bh; |
487 | 2.97M | if ah < HALF_LIMIT_2 { |
488 | 171k | m01 = m01.wrapping_add(q.wrapping_mul(m00)); |
489 | 171k | m11 = m11.wrapping_add(q.wrapping_mul(m10)); |
490 | 171k | break; |
491 | 2.80M | } |
492 | 2.80M | let q = q.wrapping_add(1); |
493 | 2.80M | m01 = m01.wrapping_add(q.wrapping_mul(m00)); |
494 | 2.80M | m11 = m11.wrapping_add(q.wrapping_mul(m10)); |
495 | | } |
496 | | } |
497 | 5.23M | bh = bh.wrapping_sub(ah); |
498 | 5.23M | if bh < HALF_LIMIT_2 { |
499 | 142k | break; |
500 | 5.09M | } |
501 | 5.09M | if ah >= bh { |
502 | 2.12M | m00 = m00.wrapping_add(m01); |
503 | 2.12M | m10 = m10.wrapping_add(m11); |
504 | 2.12M | } else { |
505 | 2.96M | let q = bh / ah; |
506 | 2.96M | bh %= ah; |
507 | 2.96M | if bh < HALF_LIMIT_2 { |
508 | 169k | m00 = m00.wrapping_add(q.wrapping_mul(m01)); |
509 | 169k | m10 = m10.wrapping_add(q.wrapping_mul(m11)); |
510 | 169k | break; |
511 | 2.80M | } |
512 | 2.80M | let q = q.wrapping_add(1); |
513 | 2.80M | m00 = m00.wrapping_add(q.wrapping_mul(m01)); |
514 | 2.80M | m10 = m10.wrapping_add(q.wrapping_mul(m11)); |
515 | | } |
516 | | } |
517 | 26.2k | } |
518 | | |
519 | | // Every entry stays below 2^63 (the half-limb refinement's exit bound, which |
520 | | // the `apply_matrix_*` overflow analyses rely on), and the column updates |
521 | | // preserve the unit determinant of the elementary step matrices. |
522 | 652k | strict_assert!(m00 < 1 << 63 && m01 < 1 << 63 && m10 < 1 << 63 && m11 < 1 << 63, "hgcd2: matrix entry reached 2^63"); |
523 | 652k | strict_assert_eq!((m00 as i128) * (m11 as i128) - (m01 as i128) * (m10 as i128), 1, "hgcd2: determinant is not 1"); |
524 | | |
525 | | // Remap M to this module's apply convention: |
526 | | // x' = m11·x - m01·y, y' = m00·y - m10·x. |
527 | 652k | Some((m11, m01, m10, m00)) |
528 | 739k | } |
529 | | |
530 | | /// `(q, r) = (n / d, n % d)` for 128-bit `n, d` with `d >= 2^64`, computed from |
531 | | /// hardware `u64` divisions (via [`div_2by1`]) instead of the `u128 / u128` |
532 | | /// software routine. The quotient is `< 2^64` since `n < 2^128 <= d * 2^64`. |
533 | | #[inline(always)] |
534 | 6.89M | fn gcd_div(n: u128, d: u128) -> (u64, u128) { |
535 | 6.89M | count!(DIVIDES += 1); |
536 | 6.89M | let (n1, n0) = ((n >> 64) as u64, n as u64); |
537 | 6.89M | let (mut d1, mut d0) = ((d >> 64) as u64, d as u64); |
538 | 6.89M | strict_assert!(d1 != 0); |
539 | 6.89M | let (q, r) = (n1 / d1, n1 % d1); |
540 | 6.89M | let (q, rem) = if q > d1 { |
541 | | // Non-normalized divisor: when `d1`'s top word is small, the single-word |
542 | | // estimate `q = n1/d1` can exceed `d1`. Rare but genuinely reachable (a |
543 | | // b-step can drop the divisor below 2^32 before an a-step divides by it), |
544 | | // so this is required for correctness, not merely defensive; marked cold |
545 | | // because it is rare. Normalize `d1` to have its top bit set, then do one |
546 | | // 128/64 division. |
547 | 12.2k | core::hint::cold_path(); |
548 | 12.2k | count!(GCD_DIV_NORM += 1); |
549 | 12.2k | let c = d1.leading_zeros(); |
550 | 12.2k | let wc = 64 - c; |
551 | 12.2k | let n2 = n1 >> wc; |
552 | 12.2k | let n1n = (n1 << c) | (n0 >> wc); |
553 | 12.2k | let n0n = n0 << c; |
554 | 12.2k | d1 = (d1 << c) | (d0 >> wc); |
555 | 12.2k | d0 <<= c; |
556 | 12.2k | let (mut q, rem1) = div_2by1(n2, n1n, d1); |
557 | 12.2k | let prod = (q as u128) * (d0 as u128); |
558 | 12.2k | let (mut t1, mut t0) = ((prod >> 64) as u64, prod as u64); |
559 | 12.2k | if t1 > rem1 || (t1 == rem1 && t0 > n0n) { |
560 | 307 | q -= 1; |
561 | 307 | let (nt0, br) = t0.overflowing_sub(d0); |
562 | 307 | t0 = nt0; |
563 | 307 | t1 = t1.wrapping_sub(d1).wrapping_sub(br as u64); |
564 | 11.9k | } |
565 | 12.2k | let (rr0, br) = n0n.overflowing_sub(t0); |
566 | 12.2k | let rr1 = rem1.wrapping_sub(t1).wrapping_sub(br as u64); |
567 | 12.2k | let rr = ((rr1 as u128) << 64) | rr0 as u128; |
568 | 12.2k | (q, rr >> c) // undo normalization |
569 | | } else { |
570 | 6.88M | let mut q = q; |
571 | 6.88M | let prod = (q as u128).wrapping_mul(d0 as u128); |
572 | 6.88M | let (mut t1, mut t0) = ((prod >> 64) as u64, prod as u64); |
573 | 6.88M | if t1 > r || (t1 == r && t0 > n0) { |
574 | 3.97k | q -= 1; |
575 | 3.97k | let (nt0, br) = t0.overflowing_sub(d0); |
576 | 3.97k | t0 = nt0; |
577 | 3.97k | t1 = t1.wrapping_sub(d1).wrapping_sub(br as u64); |
578 | 6.87M | } |
579 | 6.88M | let (rr0, br) = n0.overflowing_sub(t0); |
580 | 6.88M | let rr1 = r.wrapping_sub(t1).wrapping_sub(br as u64); |
581 | 6.88M | (q, ((rr1 as u128) << 64) | rr0 as u128) |
582 | | }; |
583 | | // Exact reconstruction: cannot wrap for a correct result since q·d + r = n < 2^128. |
584 | 6.89M | strict_assert!(rem < d && (q as u128).wrapping_mul(d).wrapping_add(rem) == n, "gcd_div: bad quotient or remainder"); |
585 | 6.89M | (q, rem) |
586 | 6.89M | } |
587 | | |
588 | | /// Multiply-accumulate: `acc + m·w + carry`, returning `(low 64 bits, high 64 |
589 | | /// bits)`. The high word is the carry into the next limb. A single widening |
590 | | /// `64x64->128` multiply plus an add-with-carry; the carry is one word. |
591 | | #[inline(always)] |
592 | 27.0M | fn mac(acc: u64, m: u64, w: u64, carry: u64) -> (u64, u64) { |
593 | 27.0M | let t = (m as u128).wrapping_mul(w as u128).wrapping_add(acc as u128).wrapping_add(carry as u128); |
594 | 27.0M | (t as u64, (t >> 64) as u64) |
595 | 27.0M | } |
596 | | |
597 | | /// Multiply-subtract-borrow: `acc - m·w - borrow`, returning `(low 64 bits, |
598 | | /// borrow)`. The borrow is the magnitude subtracted off the next limb. It fits |
599 | | /// in one word because callers pass `m < 2^63` (the [`hgcd2`] entry bound), so |
600 | | /// `m·w + borrow < 2^127 + 2^64` and its high word stays below `2^63`. |
601 | | #[inline(always)] |
602 | 27.0M | fn msb(acc: u64, m: u64, w: u64, borrow: u64) -> (u64, u64) { |
603 | 27.0M | let sub = (m as u128).wrapping_mul(w as u128).wrapping_add(borrow as u128); |
604 | 27.0M | let (lo, b) = acc.overflowing_sub(sub as u64); |
605 | 27.0M | (lo, ((sub >> 64) as u64).wrapping_add(b as u64)) |
606 | 27.0M | } |
607 | | |
608 | | /// `(x, y) <- (a·x - b·y, d·y - c·x)`, over the `len` live limbs only (the |
609 | | /// current larger length), returning the new `(x_len, y_len)`. Both results are |
610 | | /// non-negative and fit in `len` limbs for matrices from [`hgcd2`], so |
611 | | /// the limbs above `len` (already zero) stay zero. Tracking lengths here lets |
612 | | /// passes shrink with the remainders instead of always touching all limbs. |
613 | | /// |
614 | | /// Each output uses a multiply-accumulate / multiply-subtract structure: a `mac` |
615 | | /// carry chain for the additive term and an independent `msb` borrow chain for |
616 | | /// the subtractive one, both single-word. Keeping the two chains independent |
617 | | /// shortens the loop-carried dependency versus a fused signed-`i128` accumulator, |
618 | | /// which would serialize a two-register carry plus a per-limb sign-extension |
619 | | /// every step. |
620 | | #[inline] |
621 | 652k | fn apply_matrix_xy<A: LimbArray>(x: &mut A, y: &mut A, len: usize, a: u64, b: u64, c: u64, d: u64) -> (usize, usize) { |
622 | 652k | let (x, y) = (x.as_mut_slice(), y.as_mut_slice()); |
623 | | // A single up-front bound: once the compiler knows `len` is within both |
624 | | // buffers, every `[i]` with `i < len` below is provably in-bounds, so the |
625 | | // per-limb bounds checks collapse into this one branch and no `unsafe` is |
626 | | // needed. `len` never exceeds the limb count here, so this never actually |
627 | | // panics. |
628 | 652k | assert!(len <= x.len() && x.len() == y.len(), "apply_matrix_xy: len exceeds the limb count"); |
629 | 652k | count!(APPLY_XY_LIMBS += len as u64); |
630 | | // x' = a·x - b·y: `carry_ax` accumulates a·x, `borrow_x` peels off b·y. |
631 | 652k | let mut carry_ax: u64 = 0; |
632 | 652k | let mut borrow_x: u64 = 0; |
633 | | // y' = d·y - c·x. |
634 | 652k | let mut carry_dy: u64 = 0; |
635 | 652k | let mut borrow_y: u64 = 0; |
636 | | |
637 | 13.5M | for i in 0..len { |
638 | 13.5M | let xi = x[i]; |
639 | 13.5M | let yi = y[i]; |
640 | 13.5M | |
641 | 13.5M | let (px, ncx) = mac(0, a, xi, carry_ax); |
642 | 13.5M | let (nx, nbx) = msb(px, b, yi, borrow_x); |
643 | 13.5M | carry_ax = ncx; |
644 | 13.5M | borrow_x = nbx; |
645 | 13.5M | |
646 | 13.5M | let (py, ncy) = mac(0, d, yi, carry_dy); |
647 | 13.5M | let (ny, nby) = msb(py, c, xi, borrow_y); |
648 | 13.5M | carry_dy = ncy; |
649 | 13.5M | borrow_y = nby; |
650 | 13.5M | |
651 | 13.5M | x[i] = nx; |
652 | 13.5M | y[i] = ny; |
653 | 13.5M | } |
654 | | // Non-negative result fitting in `len` limbs: the positive carry-out exactly |
655 | | // cancels the negative borrow-out (both name the same high limb, which is 0). |
656 | 652k | strict_assert_eq!(carry_ax, borrow_x, "apply_matrix_xy: x went negative or overflowed len limbs"); |
657 | 652k | strict_assert_eq!(carry_dy, borrow_y, "apply_matrix_xy: y went negative or overflowed len limbs"); |
658 | | |
659 | 1.24M | let nlx = x[..len].iter().rposition(|&w| w != 0).map_or(0, |i| i + 1); _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer15apply_matrix_xyAyj30_E0CsgsKBxWKCyBU_5u3072 Line | Count | Source | 659 | 1.24M | let nlx = x[..len].iter().rposition(|&w| w != 0).map_or(0, |i| i + 1); |
Unexecuted instantiation: _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer15apply_matrix_xypE0B6_ _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer15apply_matrix_xyAyj30_Es_0CsgsKBxWKCyBU_5u3072 Line | Count | Source | 659 | 652k | let nlx = x[..len].iter().rposition(|&w| w != 0).map_or(0, |i| i + 1); |
Unexecuted instantiation: _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer15apply_matrix_xypEs_0B6_ |
660 | 1.24M | let nly = y[..len].iter().rposition(|&w| w != 0).map_or(0, |i| i + 1); _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer15apply_matrix_xyAyj30_Es0_0CsgsKBxWKCyBU_5u3072 Line | Count | Source | 660 | 1.24M | let nly = y[..len].iter().rposition(|&w| w != 0).map_or(0, |i| i + 1); |
Unexecuted instantiation: _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer15apply_matrix_xypEs0_0B6_ _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer15apply_matrix_xyAyj30_Es1_0CsgsKBxWKCyBU_5u3072 Line | Count | Source | 660 | 652k | let nly = y[..len].iter().rposition(|&w| w != 0).map_or(0, |i| i + 1); |
Unexecuted instantiation: _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer15apply_matrix_xypEs1_0B6_ |
661 | 652k | (nlx, nly) |
662 | 652k | } _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer15apply_matrix_xyAyj30_ECsgsKBxWKCyBU_5u3072 Line | Count | Source | 621 | 652k | fn apply_matrix_xy<A: LimbArray>(x: &mut A, y: &mut A, len: usize, a: u64, b: u64, c: u64, d: u64) -> (usize, usize) { | 622 | 652k | let (x, y) = (x.as_mut_slice(), y.as_mut_slice()); | 623 | | // A single up-front bound: once the compiler knows `len` is within both | 624 | | // buffers, every `[i]` with `i < len` below is provably in-bounds, so the | 625 | | // per-limb bounds checks collapse into this one branch and no `unsafe` is | 626 | | // needed. `len` never exceeds the limb count here, so this never actually | 627 | | // panics. | 628 | 652k | assert!(len <= x.len() && x.len() == y.len(), "apply_matrix_xy: len exceeds the limb count"); | 629 | 652k | count!(APPLY_XY_LIMBS += len as u64); | 630 | | // x' = a·x - b·y: `carry_ax` accumulates a·x, `borrow_x` peels off b·y. | 631 | 652k | let mut carry_ax: u64 = 0; | 632 | 652k | let mut borrow_x: u64 = 0; | 633 | | // y' = d·y - c·x. | 634 | 652k | let mut carry_dy: u64 = 0; | 635 | 652k | let mut borrow_y: u64 = 0; | 636 | | | 637 | 13.5M | for i in 0..len { | 638 | 13.5M | let xi = x[i]; | 639 | 13.5M | let yi = y[i]; | 640 | 13.5M | | 641 | 13.5M | let (px, ncx) = mac(0, a, xi, carry_ax); | 642 | 13.5M | let (nx, nbx) = msb(px, b, yi, borrow_x); | 643 | 13.5M | carry_ax = ncx; | 644 | 13.5M | borrow_x = nbx; | 645 | 13.5M | | 646 | 13.5M | let (py, ncy) = mac(0, d, yi, carry_dy); | 647 | 13.5M | let (ny, nby) = msb(py, c, xi, borrow_y); | 648 | 13.5M | carry_dy = ncy; | 649 | 13.5M | borrow_y = nby; | 650 | 13.5M | | 651 | 13.5M | x[i] = nx; | 652 | 13.5M | y[i] = ny; | 653 | 13.5M | } | 654 | | // Non-negative result fitting in `len` limbs: the positive carry-out exactly | 655 | | // cancels the negative borrow-out (both name the same high limb, which is 0). | 656 | 652k | strict_assert_eq!(carry_ax, borrow_x, "apply_matrix_xy: x went negative or overflowed len limbs"); | 657 | 652k | strict_assert_eq!(carry_dy, borrow_y, "apply_matrix_xy: y went negative or overflowed len limbs"); | 658 | | | 659 | 652k | let nlx = x[..len].iter().rposition(|&w| w != 0).map_or(0, |i| i + 1); | 660 | 652k | let nly = y[..len].iter().rposition(|&w| w != 0).map_or(0, |i| i + 1); | 661 | 652k | (nlx, nly) | 662 | 652k | } |
Unexecuted instantiation: _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer15apply_matrix_xypEB4_ |
663 | | |
664 | | /// `(t0, t1) <- (a·t0 + b·t1, c·t0 + d·t1)`, growing by at most one limb. |
665 | | /// All-positive (the cofactor matrix uses `+` signs), so `u128` accumulators |
666 | | /// suffice. Lengths are tracked inline; the cell at `max_len` is always written |
667 | | /// (carry or zero) to keep `t[len..] == 0` without a separate clearing pass. |
668 | | #[allow(clippy::too_many_arguments)] |
669 | | #[inline] |
670 | 652k | fn apply_matrix_t<C: LimbArray>(t0: &mut C, t1: &mut C, t0_len: &mut usize, t1_len: &mut usize, a: u64, b: u64, c: u64, d: u64) { |
671 | 652k | let (t0, t1) = (t0.as_mut_slice(), t1.as_mut_slice()); |
672 | 652k | let max_len = (*t0_len).max(*t1_len); |
673 | 652k | count!(APPLY_T_LIMBS += max_len as u64); |
674 | | // Single up-front bound; see `apply_matrix_xy`. With `max_len` known to be |
675 | | // within both buffers, the body `[i]` (i < max_len), the carry write |
676 | | // `[max_len]`, and the length scan (`max_len + 1` limbs) are all provably |
677 | | // in-bounds, collapsing the per-limb checks into this one branch. The |
678 | | // cofactor magnitude is bounded by the modulus with 8 limbs of headroom, |
679 | | // so the bound always holds. |
680 | 652k | assert!(max_len < t0.len() && t0.len() == t1.len(), "cofactor exceeded the cofactor buffer"); |
681 | 652k | let (a, b, c, d) = (a as u128, b as u128, c as u128, d as u128); |
682 | | |
683 | 652k | let mut c0: u128 = 0; |
684 | 652k | let mut c1: u128 = 0; |
685 | | |
686 | 18.4M | for i in 0..max_len { |
687 | 18.4M | let ti0 = t0[i] as u128; |
688 | 18.4M | let ti1 = t1[i] as u128; |
689 | 18.4M | |
690 | 18.4M | let p_at0 = a.wrapping_mul(ti0); |
691 | 18.4M | let p_bt1 = b.wrapping_mul(ti1); |
692 | 18.4M | let p_ct0 = c.wrapping_mul(ti0); |
693 | 18.4M | let p_dt1 = d.wrapping_mul(ti1); |
694 | 18.4M | |
695 | 18.4M | let n0 = c0.wrapping_add(p_at0).wrapping_add(p_bt1); |
696 | 18.4M | let n1 = c1.wrapping_add(p_ct0).wrapping_add(p_dt1); |
697 | 18.4M | |
698 | 18.4M | t0[i] = n0 as u64; |
699 | 18.4M | t1[i] = n1 as u64; |
700 | 18.4M | c0 = n0 >> 64; |
701 | 18.4M | c1 = n1 >> 64; |
702 | 18.4M | } |
703 | | // Carry-out (provably < 2^64); also clears any stale cell at `max_len`. |
704 | 652k | strict_assert!(c0 >> 64 == 0 && c1 >> 64 == 0, "apply_matrix_t: carry-out exceeded one limb"); |
705 | 652k | t0[max_len] = c0 as u64; |
706 | 652k | t1[max_len] = c1 as u64; |
707 | | |
708 | 725k | let nl0 = t0[..max_len + 1].iter().rposition(|&w| w != 0).map_or(0, |i| i + 1); _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer14apply_matrix_tAyj38_E0CsgsKBxWKCyBU_5u3072 Line | Count | Source | 708 | 725k | let nl0 = t0[..max_len + 1].iter().rposition(|&w| w != 0).map_or(0, |i| i + 1); |
Unexecuted instantiation: _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer14apply_matrix_tpE0B6_ _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer14apply_matrix_tAyj38_Es_0CsgsKBxWKCyBU_5u3072 Line | Count | Source | 708 | 652k | let nl0 = t0[..max_len + 1].iter().rposition(|&w| w != 0).map_or(0, |i| i + 1); |
Unexecuted instantiation: _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer14apply_matrix_tpEs_0B6_ |
709 | 724k | let nl1 = t1[..max_len + 1].iter().rposition(|&w| w != 0).map_or(0, |i| i + 1); _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer14apply_matrix_tAyj38_Es0_0CsgsKBxWKCyBU_5u3072 Line | Count | Source | 709 | 724k | let nl1 = t1[..max_len + 1].iter().rposition(|&w| w != 0).map_or(0, |i| i + 1); |
Unexecuted instantiation: _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer14apply_matrix_tpEs0_0B6_ _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer14apply_matrix_tAyj38_Es1_0CsgsKBxWKCyBU_5u3072 Line | Count | Source | 709 | 652k | let nl1 = t1[..max_len + 1].iter().rposition(|&w| w != 0).map_or(0, |i| i + 1); |
Unexecuted instantiation: _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer14apply_matrix_tpEs1_0B6_ |
710 | 652k | *t0_len = nl0.max(1); |
711 | 652k | *t1_len = nl1.max(1); |
712 | 652k | } _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer14apply_matrix_tAyj38_ECsgsKBxWKCyBU_5u3072 Line | Count | Source | 670 | 652k | fn apply_matrix_t<C: LimbArray>(t0: &mut C, t1: &mut C, t0_len: &mut usize, t1_len: &mut usize, a: u64, b: u64, c: u64, d: u64) { | 671 | 652k | let (t0, t1) = (t0.as_mut_slice(), t1.as_mut_slice()); | 672 | 652k | let max_len = (*t0_len).max(*t1_len); | 673 | 652k | count!(APPLY_T_LIMBS += max_len as u64); | 674 | | // Single up-front bound; see `apply_matrix_xy`. With `max_len` known to be | 675 | | // within both buffers, the body `[i]` (i < max_len), the carry write | 676 | | // `[max_len]`, and the length scan (`max_len + 1` limbs) are all provably | 677 | | // in-bounds, collapsing the per-limb checks into this one branch. The | 678 | | // cofactor magnitude is bounded by the modulus with 8 limbs of headroom, | 679 | | // so the bound always holds. | 680 | 652k | assert!(max_len < t0.len() && t0.len() == t1.len(), "cofactor exceeded the cofactor buffer"); | 681 | 652k | let (a, b, c, d) = (a as u128, b as u128, c as u128, d as u128); | 682 | | | 683 | 652k | let mut c0: u128 = 0; | 684 | 652k | let mut c1: u128 = 0; | 685 | | | 686 | 18.4M | for i in 0..max_len { | 687 | 18.4M | let ti0 = t0[i] as u128; | 688 | 18.4M | let ti1 = t1[i] as u128; | 689 | 18.4M | | 690 | 18.4M | let p_at0 = a.wrapping_mul(ti0); | 691 | 18.4M | let p_bt1 = b.wrapping_mul(ti1); | 692 | 18.4M | let p_ct0 = c.wrapping_mul(ti0); | 693 | 18.4M | let p_dt1 = d.wrapping_mul(ti1); | 694 | 18.4M | | 695 | 18.4M | let n0 = c0.wrapping_add(p_at0).wrapping_add(p_bt1); | 696 | 18.4M | let n1 = c1.wrapping_add(p_ct0).wrapping_add(p_dt1); | 697 | 18.4M | | 698 | 18.4M | t0[i] = n0 as u64; | 699 | 18.4M | t1[i] = n1 as u64; | 700 | 18.4M | c0 = n0 >> 64; | 701 | 18.4M | c1 = n1 >> 64; | 702 | 18.4M | } | 703 | | // Carry-out (provably < 2^64); also clears any stale cell at `max_len`. | 704 | 652k | strict_assert!(c0 >> 64 == 0 && c1 >> 64 == 0, "apply_matrix_t: carry-out exceeded one limb"); | 705 | 652k | t0[max_len] = c0 as u64; | 706 | 652k | t1[max_len] = c1 as u64; | 707 | | | 708 | 652k | let nl0 = t0[..max_len + 1].iter().rposition(|&w| w != 0).map_or(0, |i| i + 1); | 709 | 652k | let nl1 = t1[..max_len + 1].iter().rposition(|&w| w != 0).map_or(0, |i| i + 1); | 710 | 652k | *t0_len = nl0.max(1); | 711 | 652k | *t1_len = nl1.max(1); | 712 | 652k | } |
Unexecuted instantiation: _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer14apply_matrix_tpEB4_ |
713 | | |
714 | | /// `t0 += q · t1`, where `q` is up to `N` limbs (the exact Euclidean quotient). |
715 | 88.4k | fn t_add_qmul<C: LimbArray>(t0: &mut C, t0_len: &mut usize, q: &[u64], t1: &C, t1_len: usize) { |
716 | 88.4k | let (t0, t1) = (t0.as_mut_slice(), t1.as_slice()); |
717 | 88.4k | let cap = t0.len(); |
718 | 4.24M | for (i, &qi) in q.iter().enumerate() { |
719 | 4.24M | if qi == 0 { |
720 | 3.75M | continue; |
721 | 491k | } |
722 | 491k | let qi = qi as u128; |
723 | 491k | let mut carry: u64 = 0; |
724 | 3.14M | for (j, &t1j) in t1.iter().enumerate().take(t1_len) { |
725 | 3.14M | let k = i + j; |
726 | 3.14M | if k >= cap { |
727 | 0 | strict_assert_eq!(carry, 0, "t_add_qmul: truncated nonzero limb"); |
728 | 0 | break; |
729 | 3.14M | } |
730 | 3.14M | let prod = qi.wrapping_mul(t1j as u128).wrapping_add(t0[k] as u128).wrapping_add(carry as u128); |
731 | 3.14M | t0[k] = prod as u64; |
732 | 3.14M | carry = (prod >> 64) as u64; |
733 | | } |
734 | 491k | let mut k = i + t1_len; |
735 | 729k | while carry > 0 && k < cap { |
736 | 237k | let (s, c) = t0[k].overflowing_add(carry); |
737 | 237k | t0[k] = s; |
738 | 237k | carry = c as u64; |
739 | 237k | k += 1; |
740 | 237k | } |
741 | 491k | strict_assert!(carry == 0, "t_add_qmul: carry lost off the top"); |
742 | | } |
743 | 88.4k | *t0_len = top_len_t(t0); |
744 | 88.4k | } _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer10t_add_qmulAyj38_ECsgsKBxWKCyBU_5u3072 Line | Count | Source | 715 | 88.4k | fn t_add_qmul<C: LimbArray>(t0: &mut C, t0_len: &mut usize, q: &[u64], t1: &C, t1_len: usize) { | 716 | 88.4k | let (t0, t1) = (t0.as_mut_slice(), t1.as_slice()); | 717 | 88.4k | let cap = t0.len(); | 718 | 4.24M | for (i, &qi) in q.iter().enumerate() { | 719 | 4.24M | if qi == 0 { | 720 | 3.75M | continue; | 721 | 491k | } | 722 | 491k | let qi = qi as u128; | 723 | 491k | let mut carry: u64 = 0; | 724 | 3.14M | for (j, &t1j) in t1.iter().enumerate().take(t1_len) { | 725 | 3.14M | let k = i + j; | 726 | 3.14M | if k >= cap { | 727 | 0 | strict_assert_eq!(carry, 0, "t_add_qmul: truncated nonzero limb"); | 728 | 0 | break; | 729 | 3.14M | } | 730 | 3.14M | let prod = qi.wrapping_mul(t1j as u128).wrapping_add(t0[k] as u128).wrapping_add(carry as u128); | 731 | 3.14M | t0[k] = prod as u64; | 732 | 3.14M | carry = (prod >> 64) as u64; | 733 | | } | 734 | 491k | let mut k = i + t1_len; | 735 | 729k | while carry > 0 && k < cap { | 736 | 237k | let (s, c) = t0[k].overflowing_add(carry); | 737 | 237k | t0[k] = s; | 738 | 237k | carry = c as u64; | 739 | 237k | k += 1; | 740 | 237k | } | 741 | 491k | strict_assert!(carry == 0, "t_add_qmul: carry lost off the top"); | 742 | | } | 743 | 88.4k | *t0_len = top_len_t(t0); | 744 | 88.4k | } |
Unexecuted instantiation: _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer10t_add_qmulpEB4_ |
745 | | |
746 | | /// `out += w · t` (multi-limb unsigned), used by the single-limb tail. |
747 | 43.4k | fn add_mul_word<C: LimbArray>(out: &mut C, out_len: &mut usize, w: u64, t: &C, t_len: usize) { |
748 | 43.4k | let (out, t) = (out.as_mut_slice(), t.as_slice()); |
749 | 43.4k | let cap = out.len(); |
750 | 43.4k | if w != 0 { |
751 | 43.4k | let w = w as u128; |
752 | 43.4k | let mut carry: u64 = 0; |
753 | 2.06M | for j in 0..t_len { |
754 | | // `j < t_len <= cap` always, but the bound also lets the compiler |
755 | | // clamp the trip count and skip the per-limb bounds check. |
756 | 2.06M | if j >= cap { |
757 | 0 | break; |
758 | 2.06M | } |
759 | 2.06M | let prod = w.wrapping_mul(t[j] as u128).wrapping_add(out[j] as u128).wrapping_add(carry as u128); |
760 | 2.06M | out[j] = prod as u64; |
761 | 2.06M | carry = (prod >> 64) as u64; |
762 | | } |
763 | 43.4k | let mut k = t_len; |
764 | 44.8k | while carry > 0 && k < cap { |
765 | 1.42k | let (s, c) = out[k].overflowing_add(carry); |
766 | 1.42k | out[k] = s; |
767 | 1.42k | carry = c as u64; |
768 | 1.42k | k += 1; |
769 | 1.42k | } |
770 | 43.4k | strict_assert!(carry == 0, "add_mul_word: carry lost off the top"); |
771 | 2 | } |
772 | 43.4k | *out_len = top_len_t(out); |
773 | 43.4k | } _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer12add_mul_wordAyj38_ECsgsKBxWKCyBU_5u3072 Line | Count | Source | 747 | 43.4k | fn add_mul_word<C: LimbArray>(out: &mut C, out_len: &mut usize, w: u64, t: &C, t_len: usize) { | 748 | 43.4k | let (out, t) = (out.as_mut_slice(), t.as_slice()); | 749 | 43.4k | let cap = out.len(); | 750 | 43.4k | if w != 0 { | 751 | 43.4k | let w = w as u128; | 752 | 43.4k | let mut carry: u64 = 0; | 753 | 2.06M | for j in 0..t_len { | 754 | | // `j < t_len <= cap` always, but the bound also lets the compiler | 755 | | // clamp the trip count and skip the per-limb bounds check. | 756 | 2.06M | if j >= cap { | 757 | 0 | break; | 758 | 2.06M | } | 759 | 2.06M | let prod = w.wrapping_mul(t[j] as u128).wrapping_add(out[j] as u128).wrapping_add(carry as u128); | 760 | 2.06M | out[j] = prod as u64; | 761 | 2.06M | carry = (prod >> 64) as u64; | 762 | | } | 763 | 43.4k | let mut k = t_len; | 764 | 44.8k | while carry > 0 && k < cap { | 765 | 1.42k | let (s, c) = out[k].overflowing_add(carry); | 766 | 1.42k | out[k] = s; | 767 | 1.42k | carry = c as u64; | 768 | 1.42k | k += 1; | 769 | 1.42k | } | 770 | 43.4k | strict_assert!(carry == 0, "add_mul_word: carry lost off the top"); | 771 | 2 | } | 772 | 43.4k | *out_len = top_len_t(out); | 773 | 43.4k | } |
Unexecuted instantiation: _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer12add_mul_wordpEB4_ |
774 | | |
775 | | /// `(q, r) = (x / y, x % y)`. Fast path for a single-limb divisor (the common |
776 | | /// case once Lehmer has shrunk `y`); otherwise the uint type's own |
777 | | /// [`LehmerOps::div_rem`]. |
778 | 88.4k | fn div_rem_step<U: LehmerOps>(x: &U::Limbs, y: &U::Limbs) -> (U::Limbs, U::Limbs) { |
779 | 317k | if y.as_slice()[1..].iter().all(|&w| w == 0) {_RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer12div_rem_stepNtB6_8Uint3072E0CsgsKBxWKCyBU_5u3072 Line | Count | Source | 779 | 317k | if y.as_slice()[1..].iter().all(|&w| w == 0) { |
Unexecuted instantiation: _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer12div_rem_steppE0B6_ |
780 | 1.03k | let (q, r0) = div_n_by_1(x, y.as_slice()[0]); |
781 | 1.03k | return (q, from_word(r0)); |
782 | 87.3k | } |
783 | 87.3k | let (q, r) = U::from_limbs(*x).div_rem(U::from_limbs(*y)); |
784 | 87.3k | (q.into_limbs(), r.into_limbs()) |
785 | 88.4k | } _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer12div_rem_stepNtB4_8Uint3072ECsgsKBxWKCyBU_5u3072 Line | Count | Source | 778 | 88.4k | fn div_rem_step<U: LehmerOps>(x: &U::Limbs, y: &U::Limbs) -> (U::Limbs, U::Limbs) { | 779 | 88.4k | if y.as_slice()[1..].iter().all(|&w| w == 0) { | 780 | 1.03k | let (q, r0) = div_n_by_1(x, y.as_slice()[0]); | 781 | 1.03k | return (q, from_word(r0)); | 782 | 87.3k | } | 783 | 87.3k | let (q, r) = U::from_limbs(*x).div_rem(U::from_limbs(*y)); | 784 | 87.3k | (q.into_limbs(), r.into_limbs()) | 785 | 88.4k | } |
Unexecuted instantiation: _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer12div_rem_steppEB4_ |
786 | | |
787 | | /// `(q, r) = (x / d, x % d)` for a nonzero single-limb divisor `d` (the common |
788 | | /// case once Lehmer has collapsed `y`). The divisor is constant across all |
789 | | /// limbs, so a reciprocal is computed once and each limb costs two multiplies |
790 | | /// instead of a hardware divide. Precomputed-reciprocal division |
791 | | /// (Moller-Granlund, 2011). |
792 | 1.03k | fn div_n_by_1<A: LimbArray>(x: &A, d: u64) -> (A, u64) { |
793 | 1.03k | strict_assert!(d != 0); |
794 | 1.03k | let x = x.as_slice(); |
795 | 1.03k | let n = x.len(); |
796 | 1.03k | let mut q = A::ZERO; |
797 | 1.03k | let qs = q.as_mut_slice(); |
798 | 1.03k | let s = d.leading_zeros(); |
799 | 1.03k | let r = if s == 0 { |
800 | | // `d` already has its top bit set: no normalization shift needed. |
801 | 274 | let v = invert_limb(d); |
802 | 274 | let mut r: u64 = 0; |
803 | 13.1k | for i in (0..n).rev() { |
804 | 13.1k | (qs[i], r) = div_2by1_preinv(r, x[i], d, v); |
805 | 13.1k | } |
806 | 274 | r |
807 | | } else { |
808 | | // Normalize: divide `x << s` by `d << s` (same quotient); the bits shifted |
809 | | // off the top of `x` seed the running remainder, and `x % d = r >> s`. |
810 | 765 | let d_norm = d << s; |
811 | 765 | let v = invert_limb(d_norm); |
812 | 765 | let mut r = x[n - 1] >> (64 - s); |
813 | 36.7k | for i in (0..n).rev() { |
814 | 36.7k | let lo = if i == 0 { 0 } else { x[i - 1] }; |
815 | 36.7k | let cur = (x[i] << s) | (lo >> (64 - s)); |
816 | 36.7k | (qs[i], r) = div_2by1_preinv(r, cur, d_norm, v); |
817 | | } |
818 | 765 | r >> s |
819 | | }; |
820 | 1.03k | strict_assert!(r < d, "div_n_by_1: remainder not below the divisor"); |
821 | 1.03k | (q, r) |
822 | 1.03k | } _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer10div_n_by_1Ayj30_ECsgsKBxWKCyBU_5u3072 Line | Count | Source | 792 | 1.03k | fn div_n_by_1<A: LimbArray>(x: &A, d: u64) -> (A, u64) { | 793 | 1.03k | strict_assert!(d != 0); | 794 | 1.03k | let x = x.as_slice(); | 795 | 1.03k | let n = x.len(); | 796 | 1.03k | let mut q = A::ZERO; | 797 | 1.03k | let qs = q.as_mut_slice(); | 798 | 1.03k | let s = d.leading_zeros(); | 799 | 1.03k | let r = if s == 0 { | 800 | | // `d` already has its top bit set: no normalization shift needed. | 801 | 274 | let v = invert_limb(d); | 802 | 274 | let mut r: u64 = 0; | 803 | 13.1k | for i in (0..n).rev() { | 804 | 13.1k | (qs[i], r) = div_2by1_preinv(r, x[i], d, v); | 805 | 13.1k | } | 806 | 274 | r | 807 | | } else { | 808 | | // Normalize: divide `x << s` by `d << s` (same quotient); the bits shifted | 809 | | // off the top of `x` seed the running remainder, and `x % d = r >> s`. | 810 | 765 | let d_norm = d << s; | 811 | 765 | let v = invert_limb(d_norm); | 812 | 765 | let mut r = x[n - 1] >> (64 - s); | 813 | 36.7k | for i in (0..n).rev() { | 814 | 36.7k | let lo = if i == 0 { 0 } else { x[i - 1] }; | 815 | 36.7k | let cur = (x[i] << s) | (lo >> (64 - s)); | 816 | 36.7k | (qs[i], r) = div_2by1_preinv(r, cur, d_norm, v); | 817 | | } | 818 | 765 | r >> s | 819 | | }; | 820 | 1.03k | strict_assert!(r < d, "div_n_by_1: remainder not below the divisor"); | 821 | 1.03k | (q, r) | 822 | 1.03k | } |
Unexecuted instantiation: _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer10div_n_by_1pEB4_ |
823 | | |
824 | | /// Reciprocal of a normalized 64-bit divisor `d` (top bit set): |
825 | | /// `v = floor((2^128 - 1) / d) - 2^64`, the "3/2" reciprocal consumed by |
826 | | /// [`div_2by1_preinv`] (Moller-Granlund, "Improved division by invariant |
827 | | /// integers", 2011). The lone wide divide here runs once per [`div_n_by_1`] |
828 | | /// call (amortized over all `N` limbs), so it is off the per-limb hot path. |
829 | | #[inline] |
830 | 1.03k | const fn invert_limb(d: u64) -> u64 { |
831 | 1.03k | strict_assert!(d >> 63 == 1, "invert_limb requires a normalized divisor"); |
832 | | // v = floor((2^128 - 1) / d) - 2^64 = floor(((!d)·2^64 + !0) / d): subtracting |
833 | | // 2^64·d from the numerator drops the quotient by exactly 2^64, and that |
834 | | // numerator's high word `!d` is `< d` for a normalized `d`, so the quotient |
835 | | // fits a u64 and the software 128/64 [`div_2by1`] computes it -- avoiding the |
836 | | // slow `u128 / u128` (`__udivti3`). |
837 | 1.03k | div_2by1(!d, u64::MAX, d).0 |
838 | 1.03k | } |
839 | | |
840 | | /// `(nh·2^64 + nl) / d -> (q, r)` via the precomputed reciprocal `di` from |
841 | | /// [`invert_limb`]. Requires `d` normalized (top bit set) and `nh < d` (so the |
842 | | /// quotient fits a `u64`); the returned `r` satisfies `r < d`, sustaining that |
843 | | /// precondition limb to limb. One widening multiply, a branchless first |
844 | | /// correction, and a rarely-taken second. All `wrapping_*` (the workspace builds |
845 | | /// with `overflow-checks = true`). |
846 | | #[inline(always)] |
847 | 49.8k | fn div_2by1_preinv(nh: u64, nl: u64, d: u64, di: u64) -> (u64, u64) { |
848 | 49.8k | strict_assert!(d >> 63 == 1 && nh < d); |
849 | | // (qh:ql) = nh·di + (nh + 1)·2^64 + nl |
850 | 49.8k | let prod = (nh as u128).wrapping_mul(di as u128); |
851 | 49.8k | let (mut qh, ql) = ((prod >> 64) as u64, prod as u64); |
852 | 49.8k | let (ql, carry) = ql.overflowing_add(nl); |
853 | 49.8k | qh = qh.wrapping_add(nh.wrapping_add(1)).wrapping_add(carry as u64); |
854 | | |
855 | 49.8k | let mut r = nl.wrapping_sub(qh.wrapping_mul(d)); |
856 | | // First correction: if the estimate `qh` is one too high, `r` wraps above |
857 | | // `ql`; `mask` is all-ones then, folding the `-1`/`+d` fixup in branch-free. |
858 | 49.8k | let mask = 0u64.wrapping_sub((r > ql) as u64); |
859 | 49.8k | qh = qh.wrapping_add(mask); |
860 | 49.8k | r = r.wrapping_add(mask & d); |
861 | | // Second correction (rare): a still-too-small quotient leaves `r >= d`. |
862 | 49.8k | if r >= d { |
863 | 1.49k | r = r.wrapping_sub(d); |
864 | 1.49k | qh = qh.wrapping_add(1); |
865 | 48.3k | } |
866 | 49.8k | strict_assert!(r < d, "div_2by1_preinv: remainder not below the divisor"); |
867 | 49.8k | (qh, r) |
868 | 49.8k | } |
869 | | |
870 | | /// `(hi·2^64 + lo) / d -> (q, r)`. Knuth Algorithm D specialized to 128/64. |
871 | | /// Precondition: `d != 0` and `hi < d` (so the quotient fits in a `u64`). |
872 | | /// |
873 | | /// Not on the per-limb hot path (that goes through [`div_n_by_1`]'s reciprocal |
874 | | /// step, [`div_2by1_preinv`]): this generic divide is reached only once per |
875 | | /// [`div_n_by_1`] (to seed the reciprocal, via [`invert_limb`]) and from |
876 | | /// [`gcd_div`]'s rare normalization branch, so a plain software long division is |
877 | | /// fine. |
878 | | #[inline] |
879 | 13.2k | const fn div_2by1(hi: u64, lo: u64, d: u64) -> (u64, u64) { |
880 | 13.2k | strict_assert!(d != 0); |
881 | 13.2k | strict_assert!(hi < d, "div_2by1 quotient would overflow u64"); |
882 | | |
883 | 13.2k | let s = d.leading_zeros(); |
884 | 13.2k | let dn = if s == 0 { d } else { d << s }; |
885 | 13.2k | let un32 = if s == 0 { hi } else { (hi << s) | (lo >> (64 - s)) }; |
886 | 13.2k | let un10 = if s == 0 { lo } else { lo << s }; |
887 | | |
888 | 13.2k | let vn1 = dn >> 32; |
889 | 13.2k | let vn0 = dn & 0xFFFF_FFFF; |
890 | 13.2k | let un1 = un10 >> 32; |
891 | 13.2k | let un0 = un10 & 0xFFFF_FFFF; |
892 | | |
893 | 13.2k | let mut q1 = un32 / vn1; |
894 | 13.2k | let mut rhat = un32 - q1 * vn1; |
895 | 13.7k | while q1 >= (1u64 << 32) || q1 * vn0 > (rhat << 32) | un1 { |
896 | 1.37k | q1 -= 1; |
897 | 1.37k | rhat += vn1; |
898 | 1.37k | if rhat >= (1u64 << 32) { |
899 | 965 | break; |
900 | 410 | } |
901 | | } |
902 | | |
903 | 13.2k | let un21 = (un32 << 32).wrapping_add(un1).wrapping_sub(q1.wrapping_mul(dn)); |
904 | 13.2k | let mut q0 = un21 / vn1; |
905 | 13.2k | let mut rhat = un21 - q0 * vn1; |
906 | 14.7k | while q0 >= (1u64 << 32) || q0 * vn0 > (rhat << 32) | un0 { |
907 | 3.73k | q0 -= 1; |
908 | 3.73k | rhat += vn1; |
909 | 3.73k | if rhat >= (1u64 << 32) { |
910 | 2.32k | break; |
911 | 1.40k | } |
912 | | } |
913 | | |
914 | 13.2k | let r = (un21 << 32).wrapping_add(un0).wrapping_sub(q0.wrapping_mul(dn)) >> s; |
915 | 13.2k | let q = (q1 << 32) | q0; |
916 | 13.2k | strict_assert!( |
917 | 13.2k | r < d && (q as u128).wrapping_mul(d as u128).wrapping_add(r as u128) == ((hi as u128) << 64) | lo as u128, |
918 | | "div_2by1: bad quotient or remainder" |
919 | | ); |
920 | 13.2k | (q, r) |
921 | 13.2k | } |
922 | | |
923 | | /// Extended Euclidean GCD of two `u64`s: `(gcd, cx, cy)` with `cx·x + cy·y = g`. |
924 | | /// Called once per inversion, so `i128` intermediates are fine. |
925 | 21.7k | fn gcd_ext_u64(mut x: u64, mut y: u64) -> (u64, i64, i64) { |
926 | 21.7k | let (x0, y0) = (x as i128, y as i128); |
927 | 21.7k | let (mut a, mut b, mut c, mut d) = (1i128, 0i128, 0i128, 1i128); |
928 | 465k | while y != 0 { |
929 | 444k | let q = (x / y) as i128; |
930 | 444k | let r = x.wrapping_sub((q as u64).wrapping_mul(y)); |
931 | 444k | let (nc, nd) = (a.wrapping_sub(q.wrapping_mul(c)), b.wrapping_sub(q.wrapping_mul(d))); |
932 | 444k | a = c; |
933 | 444k | b = d; |
934 | 444k | c = nc; |
935 | 444k | d = nd; |
936 | 444k | x = y; |
937 | 444k | y = r; |
938 | 444k | } |
939 | | // Cofactors are bounded by the inputs (so the i64 narrowing is lossless) and |
940 | | // satisfy the Bezout identity for the returned gcd. |
941 | 21.7k | strict_assert!(i64::try_from(a).is_ok() && i64::try_from(b).is_ok(), "gcd_ext_u64: cofactor overflowed i64"); |
942 | 21.7k | strict_assert_eq!(a.wrapping_mul(x0).wrapping_add(b.wrapping_mul(y0)), x as i128, "gcd_ext_u64: Bezout identity failed"); |
943 | 21.7k | (x, a as i64, b as i64) |
944 | 21.7k | } |
945 | | |
946 | | /// Reduce a cofactor (with at most the modulus' significant limbs) into |
947 | | /// `[0, m)`. The cofactor is bounded by `m` at loop exit, so a few |
948 | | /// subtractions suffice; the `div_rem` branch is a defensive fallback. |
949 | 21.7k | fn reduce_mod<U: LehmerOps>(value: &U::Cofactor, m: &U::Limbs) -> U::Limbs { |
950 | 21.7k | let n = U::Limbs::LEN; |
951 | 173k | strict_assert!(value.as_slice()[n..].iter().all(|&w| w == 0), "cofactor exceeded the modulus width"); _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer10reduce_modNtB6_8Uint3072E0CsgsKBxWKCyBU_5u3072 Line | Count | Source | 951 | 173k | strict_assert!(value.as_slice()[n..].iter().all(|&w| w == 0), "cofactor exceeded the modulus width"); |
Unexecuted instantiation: _RNCINvNtCs8KLYHAG7u19_10kaspa_math6lehmer10reduce_modpE0B6_ |
952 | 21.7k | let mut out = U::Limbs::ZERO; |
953 | 21.7k | out.as_mut_slice().copy_from_slice(&value.as_slice()[..n]); |
954 | 21.7k | for _ in 0..8 { |
955 | 21.7k | if cmp_n(out.as_slice(), m.as_slice()).is_lt() { |
956 | 21.7k | return out; |
957 | 0 | } |
958 | 0 | out = sub_n(&out, m); |
959 | | } |
960 | 0 | U::from_limbs(out).div_rem(U::from_limbs(*m)).1.into_limbs() |
961 | 21.7k | } _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer10reduce_modNtB4_8Uint3072ECsgsKBxWKCyBU_5u3072 Line | Count | Source | 949 | 21.7k | fn reduce_mod<U: LehmerOps>(value: &U::Cofactor, m: &U::Limbs) -> U::Limbs { | 950 | 21.7k | let n = U::Limbs::LEN; | 951 | 21.7k | strict_assert!(value.as_slice()[n..].iter().all(|&w| w == 0), "cofactor exceeded the modulus width"); | 952 | 21.7k | let mut out = U::Limbs::ZERO; | 953 | 21.7k | out.as_mut_slice().copy_from_slice(&value.as_slice()[..n]); | 954 | 21.7k | for _ in 0..8 { | 955 | 21.7k | if cmp_n(out.as_slice(), m.as_slice()).is_lt() { | 956 | 21.7k | return out; | 957 | 0 | } | 958 | 0 | out = sub_n(&out, m); | 959 | | } | 960 | 0 | U::from_limbs(out).div_rem(U::from_limbs(*m)).1.into_limbs() | 961 | 21.7k | } |
Unexecuted instantiation: _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer10reduce_modpEB4_ |
962 | | |
963 | | /// Top 128-bit prefix of `x` and of `y`, both shifted left by the same amount so |
964 | | /// the prefix of the larger operand `x` has its top bit set (the normalization |
965 | | /// [`hgcd2`] expects). `y` is read at the same limb positions as `x`, so its |
966 | | /// prefix is naturally the smaller. Requires `x` the larger operand and |
967 | | /// `x_len >= 2`. |
968 | | #[inline] |
969 | 739k | fn highest_two_words_normalized(x: &[u64], y: &[u64], x_len: usize) -> (u128, u128) { |
970 | 739k | strict_assert!(x_len >= 2); |
971 | 739k | let i = x_len - 1; |
972 | 739k | let lz = x[i].leading_zeros(); |
973 | 739k | let lo_idx_ok = i >= 2; |
974 | | // Combine the limbs at positions (i, i-1, i-2) into the top 128 bits after a |
975 | | // left shift by `lz`. `arr[i]`'s `lz` leading zeros guarantee no overflow. |
976 | 1.47M | let combine = |hi: u64, mid: u64, lo: u64| -> u128 { |
977 | 1.47M | let base = ((hi as u128) << 64) | (mid as u128); |
978 | 1.47M | if lz == 0 { base } else { (base << lz) | ((lo >> (64 - lz)) as u128) } |
979 | 1.47M | }; _RNCNvNtCs8KLYHAG7u19_10kaspa_math6lehmer28highest_two_words_normalized0CsgsKBxWKCyBU_5u3072 Line | Count | Source | 976 | 1.47M | let combine = |hi: u64, mid: u64, lo: u64| -> u128 { | 977 | 1.47M | let base = ((hi as u128) << 64) | (mid as u128); | 978 | 1.47M | if lz == 0 { base } else { (base << lz) | ((lo >> (64 - lz)) as u128) } | 979 | 1.47M | }; |
Unexecuted instantiation: _RNCNvNtCs8KLYHAG7u19_10kaspa_math6lehmer28highest_two_words_normalized0B5_ |
980 | 739k | let x_lo = if lo_idx_ok { x[i - 2] } else { 0 }; |
981 | 739k | let y_lo = if lo_idx_ok { y[i - 2] } else { 0 }; |
982 | 739k | let (x_hi, y_hi) = (combine(x[i], x[i - 1], x_lo), combine(y[i], y[i - 1], y_lo)); |
983 | 739k | strict_assert!(x_hi >> 127 == 1 && y_hi <= x_hi, "highest_two_words_normalized: prefixes unnormalized or misordered"); |
984 | 739k | (x_hi, y_hi) |
985 | 739k | } |
986 | | |
987 | | /// A limb buffer holding the single word `w`. |
988 | | #[inline] |
989 | 45.5k | fn from_word<A: LimbArray>(w: u64) -> A { |
990 | 45.5k | let mut a = A::ZERO; |
991 | 45.5k | a.as_mut_slice()[0] = w; |
992 | 45.5k | a |
993 | 45.5k | } _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer9from_wordAyj30_ECsgsKBxWKCyBU_5u3072 Line | Count | Source | 989 | 23.8k | fn from_word<A: LimbArray>(w: u64) -> A { | 990 | 23.8k | let mut a = A::ZERO; | 991 | 23.8k | a.as_mut_slice()[0] = w; | 992 | 23.8k | a | 993 | 23.8k | } |
_RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer9from_wordAyj38_ECsgsKBxWKCyBU_5u3072 Line | Count | Source | 989 | 21.7k | fn from_word<A: LimbArray>(w: u64) -> A { | 990 | 21.7k | let mut a = A::ZERO; | 991 | 21.7k | a.as_mut_slice()[0] = w; | 992 | 21.7k | a | 993 | 21.7k | } |
Unexecuted instantiation: _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer9from_wordpEB4_ |
994 | | |
995 | | #[inline] |
996 | 54.3k | fn is_zero(x: &[u64]) -> bool { |
997 | 146k | x.iter().all(|&w| w == 0) _RNCNvNtCs8KLYHAG7u19_10kaspa_math6lehmer7is_zero0CsgsKBxWKCyBU_5u3072 Line | Count | Source | 997 | 146k | x.iter().all(|&w| w == 0) |
Unexecuted instantiation: _RNCNvNtCs8KLYHAG7u19_10kaspa_math6lehmer7is_zero0B5_ |
998 | 54.3k | } |
999 | | |
1000 | | #[inline] |
1001 | 130k | fn top_len(x: &[u64]) -> usize { |
1002 | 1.61M | x.iter().rposition(|&w| w != 0).map_or(0, |i| i + 1) _RNCNvNtCs8KLYHAG7u19_10kaspa_math6lehmer7top_len0CsgsKBxWKCyBU_5u3072 Line | Count | Source | 1002 | 1.61M | x.iter().rposition(|&w| w != 0).map_or(0, |i| i + 1) |
Unexecuted instantiation: _RNCNvNtCs8KLYHAG7u19_10kaspa_math6lehmer7top_len0B5_ _RNCNvNtCs8KLYHAG7u19_10kaspa_math6lehmer7top_lens_0CsgsKBxWKCyBU_5u3072 Line | Count | Source | 1002 | 130k | x.iter().rposition(|&w| w != 0).map_or(0, |i| i + 1) |
Unexecuted instantiation: _RNCNvNtCs8KLYHAG7u19_10kaspa_math6lehmer7top_lens_0B5_ |
1003 | 130k | } |
1004 | | |
1005 | | /// Significant length of a cofactor buffer; 0 maps to 1 by convention. |
1006 | | #[inline] |
1007 | 131k | fn top_len_t(x: &[u64]) -> usize { |
1008 | 4.26M | x.iter().rposition(|&w| w != 0).map_or(1, |i| i + 1) _RNCNvNtCs8KLYHAG7u19_10kaspa_math6lehmer9top_len_t0CsgsKBxWKCyBU_5u3072 Line | Count | Source | 1008 | 4.26M | x.iter().rposition(|&w| w != 0).map_or(1, |i| i + 1) |
Unexecuted instantiation: _RNCNvNtCs8KLYHAG7u19_10kaspa_math6lehmer9top_len_t0B5_ _RNCNvNtCs8KLYHAG7u19_10kaspa_math6lehmer9top_len_ts_0CsgsKBxWKCyBU_5u3072 Line | Count | Source | 1008 | 131k | x.iter().rposition(|&w| w != 0).map_or(1, |i| i + 1) |
Unexecuted instantiation: _RNCNvNtCs8KLYHAG7u19_10kaspa_math6lehmer9top_len_ts_0B5_ |
1009 | 131k | } |
1010 | | |
1011 | | #[inline] |
1012 | 10.9k | fn sub_n<A: LimbArray>(a: &A, b: &A) -> A { |
1013 | 10.9k | let mut out = A::ZERO; |
1014 | 10.9k | let (a, b, o) = (a.as_slice(), b.as_slice(), out.as_mut_slice()); |
1015 | 10.9k | let mut borrow = 0u64; |
1016 | 524k | for i in 0..o.len() { |
1017 | 524k | let (d1, b1) = a[i].overflowing_sub(b[i]); |
1018 | 524k | let (d2, b2) = d1.overflowing_sub(borrow); |
1019 | 524k | o[i] = d2; |
1020 | 524k | borrow = (b1 | b2) as u64; |
1021 | 524k | } |
1022 | 10.9k | out |
1023 | 10.9k | } _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer5sub_nAyj30_ECsgsKBxWKCyBU_5u3072 Line | Count | Source | 1012 | 10.9k | fn sub_n<A: LimbArray>(a: &A, b: &A) -> A { | 1013 | 10.9k | let mut out = A::ZERO; | 1014 | 10.9k | let (a, b, o) = (a.as_slice(), b.as_slice(), out.as_mut_slice()); | 1015 | 10.9k | let mut borrow = 0u64; | 1016 | 524k | for i in 0..o.len() { | 1017 | 524k | let (d1, b1) = a[i].overflowing_sub(b[i]); | 1018 | 524k | let (d2, b2) = d1.overflowing_sub(borrow); | 1019 | 524k | o[i] = d2; | 1020 | 524k | borrow = (b1 | b2) as u64; | 1021 | 524k | } | 1022 | 10.9k | out | 1023 | 10.9k | } |
Unexecuted instantiation: _RINvNtCs8KLYHAG7u19_10kaspa_math6lehmer5sub_npEB4_ |
1024 | | |
1025 | | /// Full-width limb compare of two equal-length buffers. |
1026 | | #[inline] |
1027 | 65.1k | fn cmp_n(a: &[u64], b: &[u64]) -> Ordering { |
1028 | 65.1k | strict_assert_eq!(a.len(), b.len()); |
1029 | 65.1k | cmp_prefix(a, b, a.len()) |
1030 | 65.1k | } |
1031 | | |
1032 | | #[inline] |
1033 | 713k | fn cmp_prefix(a: &[u64], b: &[u64], n: usize) -> Ordering { |
1034 | 874k | for i in (0..n).rev() { |
1035 | 874k | match a[i].cmp(&b[i]) { |
1036 | 160k | Ordering::Equal => continue, |
1037 | 713k | other => return other, |
1038 | | } |
1039 | | } |
1040 | 0 | Ordering::Equal |
1041 | 713k | } |
1042 | | |
1043 | | #[cfg(test)] |
1044 | | mod tests { |
1045 | | use super::*; |
1046 | | use crate::Uint3072; |
1047 | | use rand_chacha::{ |
1048 | | ChaCha8Rng, |
1049 | | rand_core::{RngCore, SeedableRng}, |
1050 | | }; |
1051 | | |
1052 | | const N: usize = 48; |
1053 | | |
1054 | | // MuHash prime: 2^3072 - 1103717. |
1055 | | const MUHASH_PRIME: Uint3072 = { |
1056 | | let mut max = Uint3072::MAX; |
1057 | | max.0[0] -= 1103717 - 1; |
1058 | | max |
1059 | | }; |
1060 | | |
1061 | | fn lehmer_inv(value: [u64; N], modulus: [u64; N]) -> Option<[u64; N]> { |
1062 | | let mut out = [0u64; N]; |
1063 | | invert::<Uint3072>(value, modulus, &mut out).then_some(out) |
1064 | | } |
1065 | | |
1066 | | // Malachite reference oracle over any LehmerOps type. The generic `Uint::mod_inverse` was |
1067 | | // removed, so this calls malachite's `Natural::mod_inverse` directly (malachite is a |
1068 | | // dev-dependency, available here). Requires `value < modulus`, like `invert`. |
1069 | | fn malachite_inv<U: LehmerOps>(value: U, modulus: U) -> Option<U> { |
1070 | | use malachite_base::num::arithmetic::traits::ModInverse; |
1071 | | use malachite_nz::natural::Natural; |
1072 | | let x = Natural::from_limbs_asc(value.into_limbs().as_slice()); |
1073 | | let m = Natural::from_limbs_asc(modulus.into_limbs().as_slice()); |
1074 | | x.mod_inverse(m).map(|inv| { |
1075 | | let mut out = U::Limbs::ZERO; |
1076 | | let limbs = inv.into_limbs_asc(); |
1077 | | out.as_mut_slice()[..limbs.len()].copy_from_slice(&limbs); |
1078 | | U::from_limbs(out) |
1079 | | }) |
1080 | | } |
1081 | | |
1082 | | fn addmod(a: Uint3072, b: Uint3072, m: Uint3072) -> Uint3072 { |
1083 | | let (res, overflow) = a.overflowing_add(b); |
1084 | | if overflow || res >= m { res.overflowing_sub(m).0 } else { res } |
1085 | | } |
1086 | | |
1087 | | /// `(a * b) mod m` via binary double-and-add (overflow-safe oracle). |
1088 | | fn mulmod(a: Uint3072, b: Uint3072, m: Uint3072) -> Uint3072 { |
1089 | | let mut result = Uint3072::ZERO; |
1090 | | let mut base = a % m; |
1091 | | let mut exp = b; |
1092 | | while !exp.is_zero() { |
1093 | | if exp.0[0] & 1 == 1 { |
1094 | | result = addmod(result, base, m); |
1095 | | } |
1096 | | base = addmod(base, base, m); |
1097 | | exp = exp >> 1; |
1098 | | } |
1099 | | result |
1100 | | } |
1101 | | |
1102 | | #[test] |
1103 | | fn matches_malachite_muhash_prime() { |
1104 | | // Lehmer must agree bit-for-bit with malachite (the reference) on the MuHash prime. |
1105 | | let mut rng = ChaCha8Rng::seed_from_u64(42); |
1106 | | let mut buf = [0u8; Uint3072::BYTES]; |
1107 | | for _ in 0..2000 { |
1108 | | rng.fill_bytes(&mut buf); |
1109 | | let v = Uint3072::from_le_bytes(buf) % MUHASH_PRIME; |
1110 | | if v.is_zero() { |
1111 | | continue; |
1112 | | } |
1113 | | let expected = malachite_inv(v, MUHASH_PRIME).unwrap(); |
1114 | | let got = lehmer_inv(v.0, MUHASH_PRIME.0).unwrap(); |
1115 | | assert_eq!(got.as_slice(), expected.0.as_slice(), "v={v}"); |
1116 | | } |
1117 | | } |
1118 | | |
1119 | | #[test] |
1120 | | fn product_is_one_muhash_prime() { |
1121 | | // v * inv(v) == 1 (mod prime), independent of any oracle. |
1122 | | let mut rng = ChaCha8Rng::seed_from_u64(99); |
1123 | | let mut buf = [0u8; Uint3072::BYTES]; |
1124 | | for _ in 0..500 { |
1125 | | rng.fill_bytes(&mut buf); |
1126 | | let v = Uint3072::from_le_bytes(buf) % MUHASH_PRIME; |
1127 | | if v.is_zero() { |
1128 | | continue; |
1129 | | } |
1130 | | let inv = lehmer_inv(v.0, MUHASH_PRIME.0).unwrap(); |
1131 | | assert_eq!(mulmod(v, Uint3072(inv), MUHASH_PRIME), Uint3072::from_u64(1), "v={v}"); |
1132 | | } |
1133 | | } |
1134 | | |
1135 | | #[test] |
1136 | | fn small_values_force_multilimb_quotient() { |
1137 | | // Tiny `value` makes the first quotient ~48 limbs, exercising the |
1138 | | // multi-limb `div_rem` fallback and the wide `t_add_qmul` path. |
1139 | | for v_small in [1u64, 2, 3, 5, 7, 1103717, u64::MAX] { |
1140 | | let v = Uint3072::from_u64(v_small); |
1141 | | let inv = lehmer_inv(v.0, MUHASH_PRIME.0).unwrap(); |
1142 | | assert_eq!(mulmod(v, Uint3072(inv), MUHASH_PRIME), Uint3072::from_u64(1), "v={v_small}"); |
1143 | | } |
1144 | | } |
1145 | | |
1146 | | #[test] |
1147 | | fn edge_cases_muhash_prime() { |
1148 | | let one = Uint3072::from_u64(1); |
1149 | | let p_minus_1 = MUHASH_PRIME.overflowing_sub(one).0; |
1150 | | assert_eq!(lehmer_inv(one.0, MUHASH_PRIME.0).unwrap().as_slice(), one.0.as_slice()); |
1151 | | assert_eq!(lehmer_inv(p_minus_1.0, MUHASH_PRIME.0).unwrap().as_slice(), p_minus_1.0.as_slice()); |
1152 | | assert!(lehmer_inv(Uint3072::ZERO.0, MUHASH_PRIME.0).is_none()); |
1153 | | } |
1154 | | |
1155 | | #[test] |
1156 | | fn div_n_by_1_matches_uint_div() { |
1157 | | // The reciprocal n-by-1 division must agree with Uint3072::div_rem across |
1158 | | // tiny, normalized (top bit set), and arbitrary unnormalized divisors. |
1159 | | let mut rng = ChaCha8Rng::seed_from_u64(7); |
1160 | | let mut buf = [0u8; Uint3072::BYTES]; |
1161 | | for _ in 0..20000 { |
1162 | | rng.fill_bytes(&mut buf); |
1163 | | let x = Uint3072::from_le_bytes(buf); |
1164 | | let d = match rng.next_u64() % 4 { |
1165 | | 0 => 1 + (rng.next_u64() % 1000), // tiny (large shift) |
1166 | | 1 => rng.next_u64() | (1 << 63), // already normalized |
1167 | | 2 => (rng.next_u64() >> 7).max(1), // unnormalized |
1168 | | _ => rng.next_u64().max(1), // arbitrary |
1169 | | }; |
1170 | | let (q, r) = div_n_by_1(&x.0, d); |
1171 | | let mut dl = [0u64; N]; |
1172 | | dl[0] = d; |
1173 | | let (eq, er) = Uint3072(x.0).div_rem(Uint3072(dl)); |
1174 | | assert_eq!(q.as_slice(), eq.0.as_slice(), "quotient mismatch x={x} d={d}"); |
1175 | | assert_eq!(r, er.0[0], "remainder mismatch x={x} d={d}"); |
1176 | | } |
1177 | | // Boundary divisors: 1, 2, MAX, 2^63 (min normalized), 2^63-1 (max |
1178 | | // unnormalized), and the MuHash prime difference constant. |
1179 | | for &d in &[1u64, 2, 3, u64::MAX, 1 << 63, (1 << 63) - 1, 1103717] { |
1180 | | rng.fill_bytes(&mut buf); |
1181 | | let x = Uint3072::from_le_bytes(buf); |
1182 | | let (q, r) = div_n_by_1(&x.0, d); |
1183 | | let mut dl = [0u64; N]; |
1184 | | dl[0] = d; |
1185 | | let (eq, er) = Uint3072(x.0).div_rem(Uint3072(dl)); |
1186 | | assert_eq!(q.as_slice(), eq.0.as_slice(), "quotient mismatch x={x} d={d}"); |
1187 | | assert_eq!(r, er.0[0], "remainder mismatch x={x} d={d}"); |
1188 | | } |
1189 | | } |
1190 | | |
1191 | | #[test] |
1192 | | fn gcd_div_matches_u128_both_branches() { |
1193 | | // `gcd_div` computes `(n/d, n%d)` for 128-bit `n, d` with `d >= 2^64`. |
1194 | | // Cover both arms against the native `u128` oracle: the normal arm |
1195 | | // (`q = n1/d1 <= d1`) and the rare normalization arm (`q > d1`), which |
1196 | | // fires when the divisor's top word is small. The normalization arm is |
1197 | | // hit only occasionally through `invert`, so pin it directly here. |
1198 | | let oracle = |n: u128, d: u128| ((n / d) as u64, n % d); |
1199 | | |
1200 | | // Explicit cases that force `q > d1` (small divisor top word). |
1201 | | let forcing: &[(u128, u128)] = &[ |
1202 | | (u128::MAX, (1u128 << 64) | 1), // d1 = 1 |
1203 | | (u128::MAX, 1u128 << 64), // d = 2^64 |
1204 | | ((0xDEAD_BEEFu128 << 96) | 0x1234, (1u128 << 64) | u64::MAX as u128), |
1205 | | (u128::MAX, (2u128 << 64) | 7), // d1 = 2 |
1206 | | (u128::MAX - 12345, (0x1_0000u128 << 64) | 0xABCD), // d1 = 2^16 |
1207 | | ]; |
1208 | | for &(n, d) in forcing { |
1209 | | assert!((n >> 64) as u64 / (d >> 64) as u64 > (d >> 64) as u64, "case does not force q>d1"); |
1210 | | assert_eq!(gcd_div(n, d), oracle(n, d), "q>d1 arm mismatch n={n} d={d}"); |
1211 | | } |
1212 | | // Normal arm (`q <= d1`): divisor with a large top word. |
1213 | | for &(n, d) in &[(u128::MAX, (u64::MAX as u128) << 64 | 3), (1u128 << 127, (1u128 << 127) | 1)] { |
1214 | | assert!((n >> 64) as u64 / (d >> 64) as u64 <= (d >> 64) as u64, "case is not the normal arm"); |
1215 | | assert_eq!(gcd_div(n, d), oracle(n, d), "normal arm mismatch n={n} d={d}"); |
1216 | | } |
1217 | | |
1218 | | // Randomized, biased to small divisor top words so most iterations take |
1219 | | // the `q > d1` arm; every result checked bit-for-bit against `u128`. |
1220 | | let mut rng = ChaCha8Rng::seed_from_u64(2024); |
1221 | | let mut branch_hits = 0u64; |
1222 | | for _ in 0..200_000 { |
1223 | | let n = ((rng.next_u64() as u128) << 64) | rng.next_u64() as u128; |
1224 | | let d1 = (rng.next_u64() % (1 << 20)) + 1; // 1 ..= 2^20 |
1225 | | let d = ((d1 as u128) << 64) | rng.next_u64() as u128; |
1226 | | assert_eq!(gcd_div(n, d), oracle(n, d), "random gcd_div mismatch n={n} d={d}"); |
1227 | | if (n >> 64) as u64 / d1 > d1 { |
1228 | | branch_hits += 1; |
1229 | | } |
1230 | | } |
1231 | | assert!(branch_hits > 1000, "q>d1 arm under-exercised: only {branch_hits} hits"); |
1232 | | } |
1233 | | |
1234 | | #[test] |
1235 | | fn q_gt_d1_vector_matches_malachite() { |
1236 | | // A full-inverse input that drives `gcd_div` through its `q > d1` |
1237 | | // normalization arm, pinned as an explicit regression case (that arm is |
1238 | | // otherwise only hit occasionally by the random-vector tests). |
1239 | | let v: [u64; N] = [ |
1240 | | 0xbaac78a3d8d04a44, |
1241 | | 0x454126b8efd12383, |
1242 | | 0xa93bb055d701be60, |
1243 | | 0xe940f5627944ba89, |
1244 | | 0x68842c54edf3df88, |
1245 | | 0xf7c6332a4e2be869, |
1246 | | 0x396e8533e978070a, |
1247 | | 0x2703f08794977aad, |
1248 | | 0x373dc910525ad335, |
1249 | | 0x52520bee468a9073, |
1250 | | 0xf4580a27f3ca91ee, |
1251 | | 0x5de1060b3d65f732, |
1252 | | 0x4b1072d2e2cc0da1, |
1253 | | 0x4e2a032ba51f609a, |
1254 | | 0xeae5995402410005, |
1255 | | 0x885f9eb04a59ab3c, |
1256 | | 0x165446e913daa18e, |
1257 | | 0xf2b1311c3eb843c0, |
1258 | | 0x630b8232162a2fca, |
1259 | | 0xe4224b2c61bafd4e, |
1260 | | 0x25397d500e52b519, |
1261 | | 0x7cd55e35a4ac6022, |
1262 | | 0x8629892065e194c0, |
1263 | | 0x713ea3ded7bcd68c, |
1264 | | 0xf1f138b0052d1fdc, |
1265 | | 0x7974414d82958c31, |
1266 | | 0xc29243783567cf96, |
1267 | | 0xf9214af62a72ec1e, |
1268 | | 0xa1e4feb97fdbfe07, |
1269 | | 0x5c92a43f23a3989e, |
1270 | | 0x1ad829d92571a26a, |
1271 | | 0xd09dd31bf57bf618, |
1272 | | 0x00772f32162d2fd3, |
1273 | | 0x1c0dcedad1142715, |
1274 | | 0x677b857b2c9c2713, |
1275 | | 0xe64b41b0e187f9e7, |
1276 | | 0x4062f89c0309cd59, |
1277 | | 0x3edc673c30e10664, |
1278 | | 0x330b6c680b777302, |
1279 | | 0x423676df7dfd8ccf, |
1280 | | 0x347fe8b7dee8ca42, |
1281 | | 0x0fb4eb8629ef71c1, |
1282 | | 0x5e898abada4103ac, |
1283 | | 0x39820234bad59fb2, |
1284 | | 0xd780c6aeaaa89812, |
1285 | | 0x61bdb22ece416ba0, |
1286 | | 0x79f1821384ef3f44, |
1287 | | 0x803912aebf95eede, |
1288 | | ]; |
1289 | | let got = lehmer_inv(v, MUHASH_PRIME.0).unwrap(); |
1290 | | assert_eq!(got.as_slice(), malachite_inv(Uint3072(v), MUHASH_PRIME).unwrap().0.as_slice()); |
1291 | | assert_eq!(mulmod(Uint3072(v), Uint3072(got), MUHASH_PRIME), Uint3072::from_u64(1)); |
1292 | | } |
1293 | | |
1294 | | #[test] |
1295 | | fn hgcd2_none_when_top_word_below_two() { |
1296 | | // The guess returns `None` (caller falls back to one exact division step) |
1297 | | // exactly when a top word is `< 2`; otherwise it confirms a matrix. |
1298 | | assert!(hgcd2(1, 0, 5, 0).is_none()); // ah < 2 |
1299 | | assert!(hgcd2(5, 0, 0, 9).is_none()); // bh < 2 |
1300 | | assert!(hgcd2(u64::MAX, 0, u64::MAX >> 1, 0).is_some()); // well-separated -> confirms a quotient |
1301 | | } |
1302 | | |
1303 | | /// Extended-Euclid reference for u64 moduli, in i128 (all magnitudes fit). |
1304 | | fn egcd_inv_u64(v: u64, m: u64) -> Option<u64> { |
1305 | | let (mut old_r, mut r) = (m as i128, v as i128); |
1306 | | let (mut old_t, mut t) = (0i128, 1i128); |
1307 | | while r != 0 { |
1308 | | let q = old_r / r; |
1309 | | (old_r, r) = (r, old_r - q * r); |
1310 | | (old_t, t) = (t, old_t - q * t); |
1311 | | } |
1312 | | (old_r == 1).then(|| old_t.rem_euclid(m as i128) as u64) |
1313 | | } |
1314 | | |
1315 | | #[test] |
1316 | | fn u64_exhaustive_small_range() { |
1317 | | // Every (value, modulus) with modulus in [2, 512) and value in [0, 4m): |
1318 | | // agreement with the extended-Euclid oracle, covering gcd != 1 -> None |
1319 | | // and the value-reduction path. |
1320 | | for m in 2u64..512 { |
1321 | | for v in 0..m * 4 { |
1322 | | let got = v.lehmer_invert(m); |
1323 | | assert_eq!(got, egcd_inv_u64(v % m, m), "v={v} m={m}"); |
1324 | | if let Some(inv) = got { |
1325 | | assert_eq!((inv as u128 * (v % m) as u128 % m as u128) as u64, 1, "v={v} m={m}"); |
1326 | | } |
1327 | | } |
1328 | | } |
1329 | | } |
1330 | | |
1331 | | #[test] |
1332 | | fn u64_random_matches_oracle() { |
1333 | | let mut rng = ChaCha8Rng::seed_from_u64(17); |
1334 | | for _ in 0..200_000 { |
1335 | | let m = rng.next_u64().max(2); |
1336 | | let v = rng.next_u64(); |
1337 | | assert_eq!(v.lehmer_invert(m), egcd_inv_u64(v % m, m), "v={v} m={m}"); |
1338 | | } |
1339 | | } |
1340 | | |
1341 | | #[test] |
1342 | | fn u128_random_matches_malachite() { |
1343 | | // Primitive u128 impl vs malachite, mixing full-width and single-limb |
1344 | | // operands to cross the multi-limb/single-limb phase boundary. |
1345 | | let mut rng = ChaCha8Rng::seed_from_u64(23); |
1346 | | let mut sample = |wide: bool| { |
1347 | | if wide { ((rng.next_u64() as u128) << 64) | rng.next_u64() as u128 } else { rng.next_u64() as u128 } |
1348 | | }; |
1349 | | for i in 0..50_000u32 { |
1350 | | let m = sample(i % 4 != 0).max(2); |
1351 | | let v = sample(i % 3 != 0); |
1352 | | let got = v.lehmer_invert(m); |
1353 | | assert_eq!(got, malachite_inv(v % m, m), "v={v} m={m}"); |
1354 | | } |
1355 | | } |
1356 | | |
1357 | | #[test] |
1358 | | fn uint_sizes_match_malachite() { |
1359 | | // The construct_uint! impls at the other production widths, vs malachite. |
1360 | | use crate::{Uint192, Uint256, Uint320}; |
1361 | | let mut rng = ChaCha8Rng::seed_from_u64(29); |
1362 | | macro_rules! check { |
1363 | | ($ty:ty, $iters:expr) => {{ |
1364 | | let mut buf = [0u8; <$ty>::BYTES]; |
1365 | | for _ in 0..$iters { |
1366 | | rng.fill_bytes(&mut buf); |
1367 | | let m = <$ty>::from_le_bytes(buf); |
1368 | | if m < 2u64 { |
1369 | | continue; |
1370 | | } |
1371 | | rng.fill_bytes(&mut buf); |
1372 | | let v = <$ty>::from_le_bytes(buf); |
1373 | | assert_eq!(v.lehmer_invert(m), malachite_inv(v % m, m), "v={v} m={m}"); |
1374 | | } |
1375 | | }}; |
1376 | | } |
1377 | | check!(Uint192, 3000); |
1378 | | check!(Uint256, 3000); |
1379 | | check!(Uint320, 3000); |
1380 | | } |
1381 | | |
1382 | | #[test] |
1383 | | fn lehmer_invert_edge_semantics() { |
1384 | | // Totality of the trait method: degenerate moduli, unreduced values, |
1385 | | // non-coprime pairs. |
1386 | | assert_eq!(0u64.lehmer_invert(0), None); |
1387 | | assert_eq!(5u64.lehmer_invert(0), None); |
1388 | | assert_eq!(5u64.lehmer_invert(1), None); |
1389 | | assert_eq!(1u64.lehmer_invert(2), Some(1)); |
1390 | | assert_eq!(7u64.lehmer_invert(7), None); // value == modulus reduces to 0 |
1391 | | assert_eq!(9u64.lehmer_invert(7), 2u64.lehmer_invert(7)); // reduction path |
1392 | | assert_eq!(4u64.lehmer_invert(6), None); // gcd = 2 |
1393 | | assert_eq!(3u64.lehmer_invert(8), Some(3)); // even modulus, coprime value |
1394 | | assert_eq!(3u128.lehmer_invert(8), Some(3)); |
1395 | | assert_eq!(Uint3072::from_u64(5).lehmer_invert(Uint3072::from_u64(1)), None); |
1396 | | assert_eq!(Uint3072::from_u64(3).lehmer_invert(Uint3072::from_u64(8)), Some(Uint3072::from_u64(3))); |
1397 | | } |
1398 | | } |