Insert many allow active models to have different column set (#2433)

* Insert many allow active models to have different column set

* comment and fmt

* comment

* clippy

* Fixup

* Refactor

* Docs and restore old implementation

---------

Co-authored-by: Billy Chan <ccw.billy.123@gmail.com>

Cargo.toml

@@ -34,7 +34,7 @@ tracing = { version = "0.1", default-features = false, features = ["attributes",
 rust_decimal = { version = "1", default-features = false, optional = true }
 bigdecimal = { version = "0.4", default-features = false, optional = true }
 sea-orm-macros = { version = "~1.1.2", path = "sea-orm-macros", default-features = false, features = ["strum"] }
-sea-query = { version = "0.32.0", default-features = false, features = ["thread-safe", "hashable-value", "backend-mysql", "backend-postgres", "backend-sqlite"] }
+sea-query = { version = "0.32.1", default-features = false, features = ["thread-safe", "hashable-value", "backend-mysql", "backend-postgres", "backend-sqlite"] }
 sea-query-binder = { version = "0.7.0", default-features = false, optional = true }
 strum = { version = "0.26", default-features = false }
 serde = { version = "1.0", default-features = false }

src/entity/base_entity.rs

@@ -467,6 +467,51 @@ pub trait EntityTrait: EntityName {
     /// # Ok(())
     /// # }
     /// ```
+    ///
+    /// Before 1.1.3, if the active models have different column set, this method would panic.
+    /// Now, it'd attempt to fill in the missing columns with null
+    /// (which may or may not be correct, depending on whether the column is nullable):
+    ///
+    /// ```
+    /// use sea_orm::{
+    ///     entity::*,
+    ///     query::*,
+    ///     tests_cfg::{cake, cake_filling},
+    ///     DbBackend,
+    /// };
+    ///
+    /// assert_eq!(
+    ///     cake::Entity::insert_many([
+    ///         cake::ActiveModel {
+    ///             id: NotSet,
+    ///             name: Set("Apple Pie".to_owned()),
+    ///         },
+    ///         cake::ActiveModel {
+    ///             id: NotSet,
+    ///             name: Set("Orange Scone".to_owned()),
+    ///         }
+    ///     ])
+    ///     .build(DbBackend::Postgres)
+    ///     .to_string(),
+    ///     r#"INSERT INTO "cake" ("name") VALUES ('Apple Pie'), ('Orange Scone')"#,
+    /// );
+    ///
+    /// assert_eq!(
+    ///     cake_filling::Entity::insert_many([
+    ///         cake_filling::ActiveModel {
+    ///             cake_id: ActiveValue::set(2),
+    ///             filling_id: ActiveValue::NotSet,
+    ///         },
+    ///         cake_filling::ActiveModel {
+    ///             cake_id: ActiveValue::NotSet,
+    ///             filling_id: ActiveValue::set(3),
+    ///         }
+    ///     ])
+    ///     .build(DbBackend::Postgres)
+    ///     .to_string(),
+    ///     r#"INSERT INTO "cake_filling" ("cake_id", "filling_id") VALUES (2, NULL), (NULL, 3)"#,
+    /// );
+    /// ```
     fn insert_many<A, I>(models: I) -> Insert<A>
     where
         A: ActiveModelTrait<Entity = Self>,

src/query/insert.rs

@@ -3,7 +3,7 @@ use crate::{
     PrimaryKeyTrait, QueryTrait,
 };
 use core::marker::PhantomData;
-use sea_query::{Expr, InsertStatement, OnConflict, ValueTuple};
+use sea_query::{Expr, InsertStatement, Keyword, OnConflict, SimpleExpr, Value, ValueTuple};
 
 /// Performs INSERT operations on a ActiveModel
 #[derive(Debug)]
@@ -112,7 +112,7 @@ where
     ///
     /// # Panics
     ///
-    /// Panics if the column value has discrepancy across rows
+    /// Panics if the rows have different column sets from what've previously been cached in the query statement
     #[allow(clippy::should_implement_trait)]
     pub fn add<M>(mut self, m: M) -> Self
     where
@@ -149,8 +149,16 @@ where
         self
     }
 
-    /// Add many Models to Self
+    /// Add many Models to Self. This is the legacy implementation priori to `1.1.3`.
-    pub fn add_many<M, I>(mut self, models: I) -> Self
+    ///
+    /// # Panics
+    ///
+    /// Panics if the rows have different column sets
+    #[deprecated(
+        since = "1.1.3",
+        note = "Please use [`Insert::add_many`] which does not panic"
+    )]
+    pub fn add_multi<M, I>(mut self, models: I) -> Self
     where
         M: IntoActiveModel<A>,
         I: IntoIterator<Item = M>,
@@ -161,6 +169,74 @@ where
         self
     }
 
+    /// Add many Models to Self
+    pub fn add_many<M, I>(mut self, models: I) -> Self
+    where
+        M: IntoActiveModel<A>,
+        I: IntoIterator<Item = M>,
+    {
+        let mut columns: Vec<_> = <A::Entity as EntityTrait>::Column::iter()
+            .map(|_| None)
+            .collect();
+        let mut null_value: Vec<Option<Value>> =
+            std::iter::repeat(None).take(columns.len()).collect();
+        let mut all_values: Vec<Vec<SimpleExpr>> = Vec::new();
+
+        for model in models.into_iter() {
+            let mut am: A = model.into_active_model();
+            self.primary_key =
+                if !<<A::Entity as EntityTrait>::PrimaryKey as PrimaryKeyTrait>::auto_increment() {
+                    am.get_primary_key_value()
+                } else {
+                    None
+                };
+            let mut values = Vec::with_capacity(columns.len());
+            for (idx, col) in <A::Entity as EntityTrait>::Column::iter().enumerate() {
+                let av = am.take(col);
+                match av {
+                    ActiveValue::Set(value) | ActiveValue::Unchanged(value) => {
+                        columns[idx] = Some(col); // mark the column as used
+                        null_value[idx] = Some(value.as_null()); // store the null value with the correct type
+                        values.push(col.save_as(Expr::val(value))); // same as add() above
+                    }
+                    ActiveValue::NotSet => {
+                        values.push(SimpleExpr::Keyword(Keyword::Null)); // indicate a missing value
+                    }
+                }
+            }
+            all_values.push(values);
+        }
+
+        if !all_values.is_empty() {
+            // filter only used column
+            self.query.columns(columns.iter().cloned().flatten());
+
+            // flag used column
+            self.columns = columns.iter().map(Option::is_some).collect();
+        }
+
+        for values in all_values {
+            // since we've aligned the column set, this never panics
+            self.query
+                .values_panic(values.into_iter().enumerate().filter_map(|(i, v)| {
+                    if columns[i].is_some() {
+                        // only if the column is used
+                        if !matches!(v, SimpleExpr::Keyword(Keyword::Null)) {
+                            // use the value expression
+                            Some(v)
+                        } else {
+                            // use null as standin, which must be Some
+                            null_value[i].clone().map(SimpleExpr::Value)
+                        }
+                    } else {
+                        None
+                    }
+                }));
+        }
+
+        self
+    }
+
     /// On conflict
     ///
     /// on conflict do nothing
@@ -209,8 +285,7 @@ where
         self
     }
 
-    /// Allow insert statement return safely if inserting nothing.
+    /// Allow insert statement to return without error if nothing's been inserted
-    /// The database will not be affected.
     pub fn do_nothing(self) -> TryInsert<A>
     where
         A: ActiveModelTrait,
@@ -218,7 +293,7 @@ where
         TryInsert::from_insert(self)
     }
 
-    /// alias to do_nothing
+    /// Alias to `do_nothing`
     pub fn on_empty_do_nothing(self) -> TryInsert<A>
     where
         A: ActiveModelTrait,
@@ -393,8 +468,11 @@ where
 mod tests {
     use sea_query::OnConflict;
 
-    use crate::tests_cfg::cake::{self};
+    use crate::tests_cfg::{cake, cake_filling};
-    use crate::{ActiveValue, DbBackend, DbErr, EntityTrait, Insert, IntoActiveModel, QueryTrait};
+    use crate::{
+        ActiveValue, DbBackend, DbErr, EntityTrait, Insert, IntoActiveModel, NotSet, QueryTrait,
+        Set,
+    };
 
     #[test]
     fn insert_1() {
@@ -439,7 +517,7 @@ mod tests {
     }
 
     #[test]
-    fn insert_4() {
+    fn insert_many_1() {
         assert_eq!(
             Insert::<cake::ActiveModel>::new()
                 .add_many([
@@ -459,22 +537,41 @@ mod tests {
     }
 
     #[test]
-    #[should_panic(expected = "columns mismatch")]
+    fn insert_many_2() {
-    fn insert_5() {
-        let apple = cake::ActiveModel {
-            name: ActiveValue::set("Apple".to_owned()),
-            ..Default::default()
-        };
-        let orange = cake::ActiveModel {
-            id: ActiveValue::set(2),
-            name: ActiveValue::set("Orange".to_owned()),
-        };
         assert_eq!(
             Insert::<cake::ActiveModel>::new()
+                .add_many([
+                    cake::ActiveModel {
+                        id: NotSet,
+                        name: Set("Apple Pie".to_owned()),
+                    },
+                    cake::ActiveModel {
+                        id: NotSet,
+                        name: Set("Orange Scone".to_owned()),
+                    }
+                ])
+                .build(DbBackend::Postgres)
+                .to_string(),
+            r#"INSERT INTO "cake" ("name") VALUES ('Apple Pie'), ('Orange Scone')"#,
+        );
+    }
+
+    #[test]
+    fn insert_many_3() {
+        let apple = cake_filling::ActiveModel {
+            cake_id: ActiveValue::set(2),
+            filling_id: ActiveValue::NotSet,
+        };
+        let orange = cake_filling::ActiveModel {
+            cake_id: ActiveValue::NotSet,
+            filling_id: ActiveValue::set(3),
+        };
+        assert_eq!(
+            Insert::<cake_filling::ActiveModel>::new()
                 .add_many([apple, orange])
                 .build(DbBackend::Postgres)
                 .to_string(),
-            r#"INSERT INTO "cake" ("id", "name") VALUES (NULL, 'Apple'), (2, 'Orange')"#,
+            r#"INSERT INTO "cake_filling" ("cake_id", "filling_id") VALUES (2, NULL), (NULL, 3)"#,
         );
     }
 

src/tests_cfg/cake_filling_price.rs

@@ -18,7 +18,7 @@ impl EntityName for Entity {
 pub struct Model {
     pub cake_id: i32,
     pub filling_id: i32,
-    #[cfg(feature = "with-decimal")]
+    #[cfg(feature = "with-rust_decimal")]
     pub price: Decimal,
     #[sea_orm(ignore)]
     pub ignored_attr: i32,