py/objint.c: Code review of int.from_bytes().

Support signed param:
result = int.from_bytes(bytearray(),
order='big'|'little', signed=False|True)

Signed-off-by: Ihor Nehrutsa <[email protected]>

Author: IhorNehrutsa

ports/esp32/mpconfigport.h

@@ -61,8 +61,8 @@
 #define MICROPY_ENABLE_GC                   (1)
 #define MICROPY_STACK_CHECK_MARGIN          (1024)
 #define MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF (1)
-#define MICROPY_LONGINT_IMPL                (MICROPY_LONGINT_IMPL_MPZ)
-#define MICROPY_ERROR_REPORTING             (MICROPY_ERROR_REPORTING_NORMAL)
+#define MICROPY_LONGINT_IMPL                (MICROPY_LONGINT_IMPL_LONGLONG) // (MICROPY_LONGINT_IMPL_MPZ) // (MICROPY_LONGINT_IMPL_NONE) //
+#define MICROPY_ERROR_REPORTING             (MICROPY_ERROR_REPORTING_NORMAL + 1)
 #define MICROPY_WARNINGS                    (1)
 #define MICROPY_FLOAT_IMPL                  (MICROPY_FLOAT_IMPL_FLOAT)
 #define MICROPY_STREAMS_POSIX_API           (1)

py/objint.c

@@ -39,6 +39,9 @@
 #include <math.h>
 #endif
 
+#define debug_printf(...) // mp_printf(&mp_plat_print, __VA_ARGS__); mp_printf(&mp_plat_print, "\n"); // mp_printf(&mp_plat_print, " | func:%s line:%d at %s\n", __FUNCTION__, __LINE__, __FILE__);
+#define _debug_printf(...) // mp_printf(&mp_plat_print, __VA_ARGS__);
+
 // This dispatcher function is expected to be independent of the implementation of long int
 static mp_obj_t mp_obj_int_make_new(const mp_obj_type_t *type_in, size_t n_args, size_t n_kw, const mp_obj_t *args) {
     (void)type_in;
@@ -386,7 +389,7 @@ mp_obj_t mp_obj_int_binary_op_extra_cases(mp_binary_op_t op, mp_obj_t lhs_in, mp
     }
     return MP_OBJ_NULL; // op not supported
 }
-
+/*
 // this is a classmethod
 static mp_obj_t int_from_bytes(size_t n_args, const mp_obj_t *args) {
     // TODO: Support signed param (assumes signed=False at the moment)
@@ -416,6 +419,79 @@ static mp_obj_t int_from_bytes(size_t n_args, const mp_obj_t *args) {
     }
     return mp_obj_new_int_from_uint(value);
 }
+*/
+
+void *reverce_memcpy(void *dest, const void *src, size_t len) {
+    char *d = (char *)dest + len - 1;
+    const char *s = src;
+    while (len--) {
+        *d-- = *s++;
+    }
+    return dest;
+}
+
+mp_obj_t mp_obj_integer_from_bytes_impl(const bool big_endian, const bool signd, const size_t buf_len, const byte *buf) {
+    #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE
+    if ((1ULL << (buf_len * 8 - 2)) > (MP_SMALL_INT_MAX + 1)) {
+        debug_printf("//1 Result will overflow a small-int so construct a big-int");
+        debug_printf("big_endian:%d signed:%d buf_len:%d", big_endian, signd, buf_len);
+        return mp_obj_int_from_bytes_impl(big_endian, signd, buf_len, buf);
+    }
+    #endif
+
+    union {
+        mp_int_t value;
+        byte buf[sizeof(mp_uint_t)];
+    } result = {0};
+
+    if (big_endian) {
+        reverce_memcpy(&result.value, buf, sizeof(result.value) < buf_len ? sizeof(result.value) : buf_len);
+    } else { // little endian
+        memcpy(&result.value, buf, sizeof(result.value) < buf_len ? sizeof(result.value) : buf_len);
+    }
+
+    if ((signd) && (sizeof(result.value) > buf_len) && (result.buf[buf_len - 1] & 0x80)) {
+        // Sign propagation
+        // x = 2
+        // x.to_bytes(1, 'little', True) -> b'\x02'
+        // x.to_bytes(4, 'little', True) -> b'\x02\x00\x00\x00'
+        // x = -2
+        // x.to_bytes(1, 'little', True) -> b'\xFE'
+        // x.to_bytes(4, 'little', True) -> b'\xFE\xFF\xFF\xFF'
+        debug_printf("result=0x%08X", result.value);
+        for (unsigned int i = 0; i < sizeof(result.value); i++) {
+            _debug_printf("\\%02X", result.buf[i]);
+        }
+        memset(result.buf + buf_len, 0xFF, sizeof(result.value) - buf_len);
+        debug_printf("\nresult=0x%08X", result.value);
+        for (unsigned int i = 0; i < sizeof(result.value); i++) {
+            _debug_printf("\\%02X", result.buf[i]);
+        }
+        debug_printf("");
+    }
+    #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE
+    // debug_printf("big_endian:%d signed:%d buf_len:%d sizeof(result.value):%d result.value:%ld", big_endian, signd, buf_len, sizeof(result.value), result.value);
+    if (!MP_SMALL_INT_FITS(result.value)) {
+        debug_printf("//2 Result will overflow a small-int so construct a big-int");
+        return mp_obj_int_from_bytes_impl(big_endian, signd, buf_len, buf);
+    }
+    #endif
+    return mp_obj_new_int(result.value);
+}
+
+// this is a classmethod
+// result = int.from_bytes(bytearray(), order='big', signed=False)
+static mp_obj_t int_from_bytes(size_t n_args, const mp_obj_t *args) {
+    debug_printf("mp_obj_is_bool(args[3])=%d", mp_obj_is_bool(args[3]));
+    debug_printf("n_args=%d, MP_SMALL_INT_MAX=%d, MP_SMALL_INT_MIN=%d", n_args, MP_SMALL_INT_MAX, MP_SMALL_INT_MIN);
+    // get the buffer info
+    mp_buffer_info_t bufinfo;
+    mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_READ);
+    bool big_endian = n_args < 3 || args[2] != MP_OBJ_NEW_QSTR(MP_QSTR_little);
+    bool signd = (n_args > 3) && mp_obj_is_true(args[3]);
+
+    return mp_obj_integer_from_bytes_impl(big_endian, signd, bufinfo.len, bufinfo.buf);
+}
 
 static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(int_from_bytes_fun_obj, 2, 4, int_from_bytes);
 static MP_DEFINE_CONST_CLASSMETHOD_OBJ(int_from_bytes_obj, MP_ROM_PTR(&int_from_bytes_fun_obj));

py/objint.h

@@ -54,13 +54,15 @@ char *mp_obj_int_formatted(char **buf, size_t *buf_size, size_t *fmt_size, mp_co
 char *mp_obj_int_formatted_impl(char **buf, size_t *buf_size, size_t *fmt_size, mp_const_obj_t self_in,
     int base, const char *prefix, char base_char, char comma);
 mp_int_t mp_obj_int_hash(mp_obj_t self_in);
-mp_obj_t mp_obj_int_from_bytes_impl(bool big_endian, size_t len, const byte *buf);
+mp_obj_t mp_obj_integer_from_bytes_impl(bool big_endian, bool signd, size_t len, const byte *buf);
+mp_obj_t mp_obj_int_from_bytes_impl(bool big_endian, bool signd, size_t len, const byte *buf);
 // Returns true if 'self_in' fit into 'len' bytes of 'buf' without overflowing, 'buf' is truncated otherwise.
 bool mp_obj_int_to_bytes_impl(mp_obj_t self_in, bool big_endian, size_t len, byte *buf);
 int mp_obj_int_sign(mp_obj_t self_in);
 mp_obj_t mp_obj_int_unary_op(mp_unary_op_t op, mp_obj_t o_in);
 mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_in);
 mp_obj_t mp_obj_int_binary_op_extra_cases(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_in);
 mp_obj_t mp_obj_int_pow3(mp_obj_t base, mp_obj_t exponent,  mp_obj_t modulus);
+void *reverce_memcpy(void *dest, const void *src, size_t len);
 
 #endif // MICROPY_INCLUDED_PY_OBJINT_H

py/objint_longlong.c

@@ -43,6 +43,7 @@
 const mp_obj_int_t mp_sys_maxsize_obj = {{&mp_type_int}, MP_SSIZE_MAX};
 #endif
 
+/*
 mp_obj_t mp_obj_int_from_bytes_impl(bool big_endian, size_t len, const byte *buf) {
     int delta = 1;
     if (!big_endian) {
@@ -56,6 +57,47 @@ mp_obj_t mp_obj_int_from_bytes_impl(bool big_endian, size_t len, const byte *buf
     }
     return mp_obj_new_int_from_ll(value);
 }
+*/
+#define debug_printf(...) // mp_printf(&mp_plat_print, __VA_ARGS__); mp_printf(&mp_plat_print, "\n"); // mp_printf(&mp_plat_print, " | func:%s line:%d at %s\n", __FUNCTION__, __LINE__, __FILE__);
+#define _debug_printf(...) // mp_printf(&mp_plat_print, __VA_ARGS__);
+
+mp_obj_t mp_obj_int_from_bytes_impl(bool big_endian, bool signd, size_t len, const byte *buf) {
+    union {
+        mp_longint_impl_t value;
+        byte buf[sizeof(mp_longint_impl_t)];
+    } result = {0};
+
+    if (len > sizeof(result.value)) {
+        mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("long int overflow"));
+    }
+
+    if (big_endian) {
+        reverce_memcpy(&result, buf, len);
+    } else { // little endian
+        memcpy(&result, buf, len);
+    }
+
+    if ((signd) && (sizeof(sizeof(result.value)) > len) && (result.buf[len - 1] & 0x80)) {
+        // Sign propagation
+        // x = 2
+        // x.to_bytes(1, 'little', True) -> b'\x02'
+        // x.to_bytes(4, 'little', True) -> b'\x02\x00\x00\x00'
+        // x = -2
+        // x.to_bytes(1, 'little', True) -> b'\xFE'
+        // x.to_bytes(4, 'little', True) -> b'\xFE\xFF\xFF\xFF'
+        debug_printf("result=0x%08X", result.value);
+        for (unsigned int i = 0; i < sizeof(sizeof(result.value)); i++) {
+            _debug_printf("\\%02X", result.buf[i]);
+        }
+        memset(result.buf + len, 0xFF, sizeof(sizeof(result.value)) - len);
+        debug_printf("\nresult=0x%08X", result.value);
+        for (unsigned int i = 0; i < sizeof(sizeof(result.value)); i++) {
+            _debug_printf("\\%02X", result.buf[i]);
+        }
+        debug_printf("");
+    }
+    return mp_obj_new_int_from_ll(result.value);
+}
 
 bool mp_obj_int_to_bytes_impl(mp_obj_t self_in, bool big_endian, size_t len, byte *buf) {
     assert(mp_obj_is_exact_type(self_in, &mp_type_int));

py/objint_mpz.c

@@ -106,7 +106,7 @@ char *mp_obj_int_formatted_impl(char **buf, size_t *buf_size, size_t *fmt_size,
     return str;
 }
 
-mp_obj_t mp_obj_int_from_bytes_impl(bool big_endian, size_t len, const byte *buf) {
+mp_obj_t mp_obj_int_from_bytes_impl(bool big_endian, bool signd, size_t len, const byte *buf) {
     mp_obj_int_t *o = mp_obj_int_new_mpz();
     mpz_set_from_bytes(&o->mpz, big_endian, len, buf);
     return MP_OBJ_FROM_PTR(o);

py/smallint.h

@@ -60,6 +60,7 @@
 #endif
 
 #define MP_SMALL_INT_MAX ((mp_int_t)(~(MP_SMALL_INT_MIN)))
+#define MP_SMALL_UINT_FITS(n) (((n) & ~MP_SMALL_INT_POSITIVE_MASK) == 0)
 
 // https://stackoverflow.com/a/4589384/1976323
 // Number of bits in inttype_MAX, or in any (1<<k)-1 where 0 <= k < 2040

tests/basics/int_bytes.py

@@ -9,7 +9,7 @@
 
 # check that extra zero bytes don't change the internal int value
 print(int.from_bytes(bytes(20), "little") == 0)
-print(int.from_bytes(b"\x01" + bytes(20), "little") == 1)
+print(int.from_bytes(b"\x01" + bytes(7), "little") == 1)
 
 # big-endian conversion
 print((10).to_bytes(1, "big"))